Word Search in URL Error

Question

I have a million odd URLs and searches in a text file with a unique identifier. I need to open the urls and find the search queries if they are presented as 1else 0.

Input file:

"ID" "URL","SearchTerm1","Searchterm2"
"1","www.google.com","a","b"
"2","www.yahoo.com","f","g"
"3","www.att.net","k"
"4" , "www.facebook.com","cs","ee"

Code snippet:

import urllib2
import re
import csv 
import datetime 
from BeautifulSoup import BeautifulSoup

with open('txt.txt') as inputFile, open ('results.txt','w+') as proc_seqf:
        header = 'Id' + '\t' + 'URL' +  '\t'  
        for i in range(1,3):
            header += 'Found_Search' + str(i) +  '\t'
        header += '\n'
        proc_seqf.write(header)
        for line in inputFile:
            line=line.split(",")
            url = 'http://' + line[1]
            req = urllib2.Request(url, headers={'User-Agent' : "Magic Browser"})
            html_content = urllib2.urlopen(req).read()
            soup = BeautifulSoup(html_content)
            if line[2][0:1] == '"' and line[2][-1:] == '"':
                 line[2] = line[2][1:-1]
            matches = soup(text=re.compile(line[2]))
            #print soup(text=re.compile(line[2]))
            #print matches
            if len(matches) == 0 or line[2].isspace() == True:
                output_1 =0
            else:
                output_1 =1
            #print output_1
            #print line[2]
            if line[3][0:1] == '"' and line[3][-1:] == '"':
                 line[3] = line[3][1:-1]
            matches = soup(text=re.compile(line[3]))
            if len(matches) == 0 or line[3].isspace() == True:
                output_2 =0
            else:
                output_2 =1
            #print output_2
            #print line[3]

            proc_seqf.write("{}\t{}\t{}\t{}\n".format(line[0],url,output_1, output_2))

output File:

ID,SearchTerm1,Searchterm2
1,0,1
2,1,0
3,0
4,1,1

Two problems with the code:

• when i run about 200 urls it gives me urlopen error [Errno 11004] getaddrinfo failed error.

• Is there a way to search for something that closely matches but doesn't exactly match?

Answer 1

when i run about 200 urls at once, it gives me urlopen error [Errno 11004] getaddrinfo error.

, DNS URL .

, , .

- , :

try:
    html_content = urllib2.urlopen(req).read()
except urllib2.URLError as ex:
    print 'Could not fetch {} because of {}, skipping.'.format(url, ex)
    # skip the rest of the loop
    continue

, , ; , , DNS- , .
:

import time

class FetchException(Exception):
    pass

def fetch_url(req, retries=5):
    for i in range(1, retries + 1):
        try:
            html_content = urllib2.urlopen(req).read()
        except urllib2.URLError as ex:
            print 'Could not fetch {} because of {}, skipping.'.format(url, ex)
            time.sleep(1 * i))
            continue
        else:
            return html_content
     # if we reach here then all lookups have failed
     raise FetchFailedException() 

# In your main code
try:
    html_content = fetch_url(req)
except FetchFailedException:
    print 'Could not fetch {} because of {}, skipping.'.format(url, ex)
    # skip the rest of the loop
    continue

-, , ?

, ?.

docs:

RE 0 1 RE. ? "a", "ab".

>>> s = 'Abc In'
>>> m = re.match(r'Abc In.', s)
>>> m is None
True

# Surround `.` with brackets so that `?` only applies to the `.`
>>> m = re.match(r'Abc In(.)?', s)
>>> m.group()
'Abc In'
>>> m = re.match(r'Abc In(.)?', 'Abc In.')
>>> m.group()
'Abc In.'

r, . . , (\), .

, :

matches = soup(text=re.compile(r'{}(.)?').format(line[2]))