jwhitlock · June 25, 2018 21:54
diff --git a/bug_1467834.csv b/bug_1467834.csv
diff --git a/detect_1467834.py b/detect_1467834.py
 # To be pasted into a Django shell
 import csv
 from StringIO import StringIO
 from pyquery import PyQuery as pq

 docs = (Document.objects
        .exclude(is_redirect=True)
        .filter(html__icontains='embedlivesample'))
 counts = []
 for doc_id in docs.values_list('id', flat=True):
    doc = Document.objects.get(id=doc_id)
    # print("Looking at %s" % doc.get_full_url)
    html = doc.html
    page = pq(html)
    start_pos = 0
    done = False
    good, bad_id, bad_content = 0, 0, 0
    while not done:
        try:
            index = html.lower().index('embedlivesample', start_pos)
        except ValueError:
            # last one
            # print("No more macros.")
            done = True
        else:
            # print("Found one at %d: %s" % (index, html[index:index+40]))
            # Find the open parenthesis
            open_parens = html.index('(', index)
            # Find the starting quote
            quote = None
            quote_pos = open_parens + 1
            while not quote:
                char = html[quote_pos]
                assert char in (" '\"")
                if char in ("'", '"'):
                    quote = char
                else:
                    quote_pos +=1
            # Read the id, which may contain escaped quotes
            id_name = ''
            id_pos = quote_pos + 1
            skip_one = False
            last_quote = False
            while not last_quote:
                if skip_one:
                    id_name += html[id_pos]
                    skip_one = False
                else:
                    char = html[id_pos]
                    if char == '//':
                        id_name += html[id_pos]
                        skip_one = True
                    elif char == quote:
                        last_quote = True
                    else:
                        id_name += html[id_pos]
                id_pos += 1
            # print("Found id %s" % id_name)
            # Use new starting place next time
            start_pos = id_pos
            # Is it a valid ID?
            if not id_name:
                print("BAD: no id")
                bad_id += 1
            elif not pq("#" + id_name):
                # print("BAD: no section found")
                bad_id += 1
            else:
                sample = doc.extract.code_sample(id_name)
                if sample['html'] or sample['css'] or sample['js']:
                    # print("GOOD: section found")
                    good += 1
                else:
                    # print("BAD: no content in section")
                    bad_content += 1
    is_ok = (bad_id == 0) and (bad_content == 0) and (good > 0)
    counts.append((1 if is_ok else 0, doc.get_full_url(), doc.id, good, bad_id, bad_content))

 counts.sort()
 out = StringIO()
 writer = csv.writer(out)
 writer.writerow(('Is OK', 'URL', 'ID', 'Good', 'Bad IDs', 'Bad Content'))
 writer.writerows(counts)
 print out.getvalue()
	# To be pasted into a Django shell
	import csv
	from StringIO import StringIO
	from pyquery import PyQuery as pq

	docs = (Document.objects
	.exclude(is_redirect=True)
	.filter(html__icontains='embedlivesample'))
	counts = []
	for doc_id in docs.values_list('id', flat=True):
	doc = Document.objects.get(id=doc_id)
	# print("Looking at %s" % doc.get_full_url)
	html = doc.html
	page = pq(html)
	start_pos = 0
	done = False
	good, bad_id, bad_content = 0, 0, 0
	while not done:
	try:
	index = html.lower().index('embedlivesample', start_pos)
	except ValueError:
	# last one
	# print("No more macros.")
	done = True
	else:
	# print("Found one at %d: %s" % (index, html[index:index+40]))
	# Find the open parenthesis
	open_parens = html.index('(', index)
	# Find the starting quote
	quote = None
	quote_pos = open_parens + 1
	while not quote:
	char = html[quote_pos]
	assert char in (" '\"")
	if char in ("'", '"'):
	quote = char
	else:
	quote_pos +=1
	# Read the id, which may contain escaped quotes
	id_name = ''
	id_pos = quote_pos + 1
	skip_one = False
	last_quote = False
	while not last_quote:
	if skip_one:
	id_name += html[id_pos]
	skip_one = False
	else:
	char = html[id_pos]
	if char == '//':
	id_name += html[id_pos]
	skip_one = True
	elif char == quote:
	last_quote = True
	else:
	id_name += html[id_pos]
	id_pos += 1
	# print("Found id %s" % id_name)
	# Use new starting place next time
	start_pos = id_pos
	# Is it a valid ID?
	if not id_name:
	print("BAD: no id")
	bad_id += 1
	elif not pq("#" + id_name):
	# print("BAD: no section found")
	bad_id += 1
	else:
	sample = doc.extract.code_sample(id_name)
	if sample['html'] or sample['css'] or sample['js']:
	# print("GOOD: section found")
	good += 1
	else:
	# print("BAD: no content in section")
	bad_content += 1
	is_ok = (bad_id == 0) and (bad_content == 0) and (good > 0)
	counts.append((1 if is_ok else 0, doc.get_full_url(), doc.id, good, bad_id, bad_content))

	counts.sort()
	out = StringIO()
	writer = csv.writer(out)
	writer.writerow(('Is OK', 'URL', 'ID', 'Good', 'Bad IDs', 'Bad Content'))
	writer.writerows(counts)
	print out.getvalue()
No results found