benallard · May 19, 2022 07:00
diff --git a/incidence.csv b/incidence.csv
diff --git a/index.html b/index.html
 <!DOCTYPE html>
 <html>

 <head>
  <meta charset="utf-8" />
  <script src="//d3js.org/d3.v6.min.js"></script>
  <style>
    body {
      font: 12px Arial;
    }

    path.line {
      stroke: url(#line-gradient);
      stroke-width: 2;
      fill: none;
    }

    path.ref {
      stroke: url(#line-gradient);
      stroke-dasharray: 5;
      stroke-width: 2;
    }
  </style>
 </head>

 <body>
  <h1>COVID-19 7-Tage-Inzidenz im Gemeinden der Landkreis und Stadt Osnabrück</h1>
  <select id="selectButton"></select><br>
  <script>
    //"use strict";

    const ref = "Landkreis Osnabrück";
    const def = "Stadt Osnabrück";

    var margin = { top: 10, right: 30, bottom: 30, left: 60 },
      width = 960 - margin.left - margin.right,
      height = 400 - margin.top - margin.bottom;

    var parseDate = d3.timeParse("%Y-%m-%d"),
      x = d3.scaleUtc().range([0, width]),
      y = d3.scaleLinear().range([height, 0]);


    var line = d3.line().x(d => x(d.date)).y(d => y(d.incidence));

    var svg = d3.select('body')
      .append("svg")
      .attr("width", width + margin.left + margin.right)
      .attr("height", height + margin.top + margin.bottom)
      .append("g")
      .attr("transform", "translate(" + margin.left + "," + margin.top + ")");

    // Create a rect on top of the svg area: this rectangle recovers mouse position
    svg
      .append('rect')
      .attr('id', 'mouse-area')
      .style("fill", "none")
      .style("pointer-events", "all")
      .attr('width', width)
      .attr('height', height)
      .on('mouseover', mouseover)
      .on('mouseout', mouseout);

    // What happens when the mouse move -> show the annotations at the right positions.
    function mouseover() {
      focus.style("opacity", 1)
      focusText.style("opacity", 1)
    };
    function mouseout() {
      focus.style("opacity", 0)
      focusText.style("opacity", 0)
    };

    svg.append("g")
      .classed("y-axis", true);

    // Set the gradient
    var gradient = svg.append("linearGradient")
      .attr("id", "line-gradient")
      .attr("gradientUnits", "userSpaceOnUse");

    // This allows to find the closest X index of the mouse:
    var bisect = d3.bisector(d => d.date).left;

    // Create the circle that travels along the curve of chart
    var focus = svg
      .append('g')
      .append('circle')
      .style("fill", "none")
      .attr("stroke", "black")
      .attr('r', 4)
      .style("opacity", 0)

    // Create the text that travels along the curve of chart
    var focusText = svg
      .append('g')
      .append('text')
      .style("opacity", 0)
      .attr("text-anchor", "left")
      .attr("alignment-baseline", "middle")

    d3.csv("incidence.csv", row => {
      row.date = parseDate(row.day);
      return row;
    }).then(data => {

      //console.log(data);

      res = [];
      // We somehow need to transpose the data
      data.forEach(d => {
        var date = d.date;
        delete d.date;
        delete d.day;
        for (var city in d) {
          value = +d[city];
          res.push({
            city: city,
            date: date,
            incidence: value
          });
        }
      });

      d3.select('#selectButton')
        .selectAll('options')
        .data(d3.group(res, d => d.city))
        .enter().append('option')
        .text(d => d[0])
        .attr("value", d => d[0])
        .attr("selected", d => d[0] == def ? 'selected' : null);

      d3.select('#selectButton').on("change", (e) => {
        update(e.target.value);
      });

      x.domain(d3.extent(res, d => d.date));

      svg.append("g")
        .attr("transform", "translate(0, " + height + ")")
        .classed("x-axis", true)
        .call(d3.axisBottom(x));

      window.data = res;
      update(def);
    });

    function update(selected) {
      // filter data
      // the rest id data dependent
      filtered = data.filter(d => d.city == ref || d.city == selected)

      y.domain([d3.min([0, d3.min(filtered, d => d.incidence)]), d3.max(filtered, d => d.incidence)]);

      gradient.attr("x1", 0)
        .attr("y1", y(0))
        .attr("x2", 0)
        .attr("y2", y(200))
        .selectAll("stop")
        .data([
          { offset: "0%", color: "limegreen" },
          { offset: "17.5%", color: "yellow" },
          { offset: "25%", color: "red" },
          { offset: "50%", color: "darkred" },
          { offset: "100%", color: "purple" }
        ])
        .enter().append("stop")
        .attr("offset", d => d.offset)
        .attr("stop-color", d => d.color);

      svg.selectAll(".y-axis")
        .transition()
        .duration(300)
        .call(d3.axisLeft(y));

      var all = svg.selectAll("path.line")
        .data(d3.group(filtered, d => d.city));
      all
        .enter().append("path")
        .merge(all)
        .classed("line", true)
        .classed("ref", d => d[0] == ref)
        .transition()
        .duration(300)
        .attr("d", d => line(d[1]));
      all
        .exit().remove();

      d3.select('#mouse-area')
        .on("mousemove", (event) => {
          var citydata = filtered.filter(d => d.city == selected);
          // recover coordinate we need
          var x0 = x.invert(d3.pointer(event)[0]);
          var i = bisect(citydata, x0);
          selectedData = citydata[i];
          focus
            .attr("cx", x(selectedData.date))
            .attr("cy", y(selectedData.incidence));
          focusText
            .html(d3.timeFormat("%d %b %Y")(selectedData.date) + ": " + d3.format(".1f")(selectedData.incidence))
            .attr("x", x(selectedData.date) - 110)
            .attr("y", y(selectedData.incidence) + 5)
        });
    };
  </script><br>
  <p><a href="https://bl.ocks.org/benallard/6209ba9892426b7fe4b23ccab6430c51">Block</a></p>
 </body>

 </html>
diff --git a/rona-pdf.py b/rona-pdf.py
 """
 So I used to open my browser, look for the right spot in the latest pdf, enter that number in GSheet, and display a graph.
 I'm tired. let's do it again here.
 """

 import csv
 import datetime
 import json
 import locale
 import os.path

 from pdfminer.high_level import extract_pages
 from pdfminer.layout import LTTextContainer
 import requests

 # For better parsing of the values
 locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')

 FILENAME = lambda day: f"data/{day.year}-{day.month:02}-{day.day:02}.pdf"

 def download_file(day: datetime.date):
    print(f"Downloading file for {day}")
    for pattern in (
            '_dashboard_corona_presse', # usual one
            '-presse', # 2021-05-27
            '_dashboard_corona_presse_0', # 2021-06-29
            '_dashboard_corona_presse_0_0', # 2021-09-06
            ):
        url = f"https://corona-os.de/sites/default/files/{day.year}-{day.month:02}/{day.year}-{day.month:02}-{day.day:02}{pattern}.pdf"
        with requests.get(url) as r:
            if r.status_code == 404:
                continue
            if int(r.headers['Content-Length']) == 0:  # 2021-09-06
                continue
            r.raise_for_status()
            with open(FILENAME(day), 'wb') as f:
                f.write(r.content)
            return True
    print("Document not found")
    return False

 def analyse(day: datetime.date):
    """ We are actually only interested in the table in the bottom-right corner.
    Those are actually two tables, or a table on two 'columns'.
    Line 'headers' (city names) are right-justified, and the values are centered.
    On the 2021-11-05 they changed the order of the fields.
    """
    # y -> name
    cities = {}
    # city name -> [infected, quarantined]
    res = {}
    unmatched = []
    with open(FILENAME(day), "rb") as f:
        # We are only interested in the first page
        page = next(extract_pages(f))
        page_width = page.width
        page_height = page.height
        for element in page:
            if not isinstance(element, LTTextContainer):
                continue
            for line in element:
                if not hasattr(line, 'x1'):
                    continue
                if (line.x1 < page_width / 2) or (line.y0 > page_height / 2):
                    continue
                print(f"({line.x0:.0f}, {line.y0:.0f}): {line.get_text().strip()}")
                try:
                    intVal = locale.atoi(line.get_text())
                    if  not cities:
                        unmatched.append((line.y0, line.height, intVal))
                        continue
                    city = next(filter(lambda y: abs(y - line.y0) < line.height / 2, cities.keys()))
                    if city:
                        name = cities[city]
                        if res[name][0] is None:
                            res[name][0] = intVal
                        else:
                            res[name][1] = intVal
                except ValueError:
                    # Not a  number
                    name = line.get_text().strip()
                    cities[line.y0] = name
                    res[name] = [None, None]
    for yVal, heightVal, val in unmatched:
        city = next(filter(lambda y: abs(y - yVal) < heightVal / 2, cities.keys()))
        if city:
            name = cities[city]
            if res[name][0] is None:
                res[name][0] = val
            else:
                res[name][1] = val
    # remove the unmatched artefacts
    return {k: v for k, v in res.items() if v[0] is not None and v[1] is not None}

 def download_all():
    one_day = datetime.timedelta(days=1)
    day = datetime.date.today()
    last = None
    while True:
        if os.path.exists(FILENAME(day)):
            last = day
            break
        # Will break on first three non-available files (29.10.2020)
        if not download_file(day):
            if day != datetime.date.today() and (day - last).days > 3 :
                break

        last = day
        day = day - one_day

 def analyse_all():
    one_day = datetime.timedelta(days=1)
    day = datetime.date.today() + one_day
    skip = 0
    while True:
        day = day - one_day
        print(f"analysing {day}")
        # Will break on first missing file
        dayStr = day.isoformat()
        with open('data/data.json') as f:
            data = json.load(f)
        if dayStr in data:
            continue
        try:
            res = analyse(day)
        except FileNotFoundError:
            skip += 1
            if skip > 3:
                break
            continue
        data[dayStr] = res
        with open('data/data.json', 'wt') as f:
            json.dump(data, f, indent=True)

 download_all()
 analyse_all()

 def validate_data():
    with open('data/census.json') as f:
        CENSUS = json.load(f)

    with open('data/data.json') as f:
        DATA = json.load(f)

    cities = set(CENSUS.keys())

    good = True
    for k, v in DATA.items():
        entries = set(v.keys())
        if cities - entries or entries - cities:
            print (f"{k}: missing: {cities - entries}, too much: {entries - cities}")
            good = False

    return good

 class NoData(Exception):
    pass

 class DailyData(object):
    def __init__(self):
        # date -> amount
        self.data = {}

    def add(self, date, amount):
        self.data[date] = amount

    def get(self, date):
        """ returns the incidence for the day """
        lastWeek = date - datetime.timedelta(days=7)
        if lastWeek not in self.data or date not in self.data:
            raise NoData()
        return self.data[date] - self.data[lastWeek]


 def incidence():
    with open('data/census.json') as f:
        CENSUS = json.load(f)

    with open('data/data.json') as f:
        DATA = json.load(f)

    with open('data/incidence.csv', 'w', newline = '') as f:
        fieldnames = ['day'] + sorted(CENSUS.keys())
        writer = csv.DictWriter(f, fieldnames = fieldnames)

        writer.writeheader()
        # city -> dailydata
        data = {}
        for day in sorted(map(lambda s: datetime.date.fromisoformat(s), DATA.keys())):
            daydata = DATA[day.isoformat()]
            row = {'day': day.isoformat()}
            for city in CENSUS.keys():
                # 0 is amount of infected, 1 is quarantined
                # Add the new value
                data.setdefault(city, DailyData()).add(day, daydata[city][0])
                try:
                    row[city] = (data[city].get(day) / CENSUS[city]) * 100_000
                except NoData:
                    pass
            if len(row.keys()) > 1:  # day
                writer.writerow(row)

        print(row)

    
 if not validate_data():
    print("Bad, bad, bad ...")
 incidence()
	<!DOCTYPE html>
	<html>

	<head>
	<meta charset="utf-8" />
	<script src="//d3js.org/d3.v6.min.js"></script>
	<style>
	body {
	font: 12px Arial;
	}

	path.line {
	stroke: url(#line-gradient);
	stroke-width: 2;
	fill: none;
	}

	path.ref {
	stroke: url(#line-gradient);
	stroke-dasharray: 5;
	stroke-width: 2;
	}
	</style>
	</head>

	<body>
	<h1>COVID-19 7-Tage-Inzidenz im Gemeinden der Landkreis und Stadt Osnabrück</h1>
	<select id="selectButton"></select><br>
	<script>
	//"use strict";

	const ref = "Landkreis Osnabrück";
	const def = "Stadt Osnabrück";

	var margin = { top: 10, right: 30, bottom: 30, left: 60 },
	width = 960 - margin.left - margin.right,
	height = 400 - margin.top - margin.bottom;

	var parseDate = d3.timeParse("%Y-%m-%d"),
	x = d3.scaleUtc().range([0, width]),
	y = d3.scaleLinear().range([height, 0]);


	var line = d3.line().x(d => x(d.date)).y(d => y(d.incidence));

	var svg = d3.select('body')
	.append("svg")
	.attr("width", width + margin.left + margin.right)
	.attr("height", height + margin.top + margin.bottom)
	.append("g")
	.attr("transform", "translate(" + margin.left + "," + margin.top + ")");

	// Create a rect on top of the svg area: this rectangle recovers mouse position
	svg
	.append('rect')
	.attr('id', 'mouse-area')
	.style("fill", "none")
	.style("pointer-events", "all")
	.attr('width', width)
	.attr('height', height)
	.on('mouseover', mouseover)
	.on('mouseout', mouseout);

	// What happens when the mouse move -> show the annotations at the right positions.
	function mouseover() {
	focus.style("opacity", 1)
	focusText.style("opacity", 1)
	};
	function mouseout() {
	focus.style("opacity", 0)
	focusText.style("opacity", 0)
	};

	svg.append("g")
	.classed("y-axis", true);

	// Set the gradient
	var gradient = svg.append("linearGradient")
	.attr("id", "line-gradient")
	.attr("gradientUnits", "userSpaceOnUse");

	// This allows to find the closest X index of the mouse:
	var bisect = d3.bisector(d => d.date).left;

	// Create the circle that travels along the curve of chart
	var focus = svg
	.append('g')
	.append('circle')
	.style("fill", "none")
	.attr("stroke", "black")
	.attr('r', 4)
	.style("opacity", 0)

	// Create the text that travels along the curve of chart
	var focusText = svg
	.append('g')
	.append('text')
	.style("opacity", 0)
	.attr("text-anchor", "left")
	.attr("alignment-baseline", "middle")

	d3.csv("incidence.csv", row => {
	row.date = parseDate(row.day);
	return row;
	}).then(data => {

	//console.log(data);

	res = [];
	// We somehow need to transpose the data
	data.forEach(d => {
	var date = d.date;
	delete d.date;
	delete d.day;
	for (var city in d) {
	value = +d[city];
	res.push({
	city: city,
	date: date,
	incidence: value
	});
	}
	});

	d3.select('#selectButton')
	.selectAll('options')
	.data(d3.group(res, d => d.city))
	.enter().append('option')
	.text(d => d[0])
	.attr("value", d => d[0])
	.attr("selected", d => d[0] == def ? 'selected' : null);

	d3.select('#selectButton').on("change", (e) => {
	update(e.target.value);
	});

	x.domain(d3.extent(res, d => d.date));

	svg.append("g")
	.attr("transform", "translate(0, " + height + ")")
	.classed("x-axis", true)
	.call(d3.axisBottom(x));

	window.data = res;
	update(def);
	});

	function update(selected) {
	// filter data
	// the rest id data dependent
	filtered = data.filter(d => d.city == ref \|\| d.city == selected)

	y.domain([d3.min([0, d3.min(filtered, d => d.incidence)]), d3.max(filtered, d => d.incidence)]);

	gradient.attr("x1", 0)
	.attr("y1", y(0))
	.attr("x2", 0)
	.attr("y2", y(200))
	.selectAll("stop")
	.data([
	{ offset: "0%", color: "limegreen" },
	{ offset: "17.5%", color: "yellow" },
	{ offset: "25%", color: "red" },
	{ offset: "50%", color: "darkred" },
	{ offset: "100%", color: "purple" }
	])
	.enter().append("stop")
	.attr("offset", d => d.offset)
	.attr("stop-color", d => d.color);

	svg.selectAll(".y-axis")
	.transition()
	.duration(300)
	.call(d3.axisLeft(y));

	var all = svg.selectAll("path.line")
	.data(d3.group(filtered, d => d.city));
	all
	.enter().append("path")
	.merge(all)
	.classed("line", true)
	.classed("ref", d => d[0] == ref)
	.transition()
	.duration(300)
	.attr("d", d => line(d[1]));
	all
	.exit().remove();

	d3.select('#mouse-area')
	.on("mousemove", (event) => {
	var citydata = filtered.filter(d => d.city == selected);
	// recover coordinate we need
	var x0 = x.invert(d3.pointer(event)[0]);
	var i = bisect(citydata, x0);
	selectedData = citydata[i];
	focus
	.attr("cx", x(selectedData.date))
	.attr("cy", y(selectedData.incidence));
	focusText
	.html(d3.timeFormat("%d %b %Y")(selectedData.date) + ": " + d3.format(".1f")(selectedData.incidence))
	.attr("x", x(selectedData.date) - 110)
	.attr("y", y(selectedData.incidence) + 5)
	});
	};
	</script><br>
	<p><a href="https://bl.ocks.org/benallard/6209ba9892426b7fe4b23ccab6430c51">Block</a></p>
	</body>

	</html>
	"""
	So I used to open my browser, look for the right spot in the latest pdf, enter that number in GSheet, and display a graph.
	I'm tired. let's do it again here.
	"""

	import csv
	import datetime
	import json
	import locale
	import os.path

	from pdfminer.high_level import extract_pages
	from pdfminer.layout import LTTextContainer
	import requests

	# For better parsing of the values
	locale.setlocale(locale.LC_ALL, 'de_DE.UTF-8')

	FILENAME = lambda day: f"data/{day.year}-{day.month:02}-{day.day:02}.pdf"

	def download_file(day: datetime.date):
	print(f"Downloading file for {day}")
	for pattern in (
	'_dashboard_corona_presse', # usual one
	'-presse', # 2021-05-27
	'_dashboard_corona_presse_0', # 2021-06-29
	'_dashboard_corona_presse_0_0', # 2021-09-06
	):
	url = f"https://corona-os.de/sites/default/files/{day.year}-{day.month:02}/{day.year}-{day.month:02}-{day.day:02}{pattern}.pdf"
	with requests.get(url) as r:
	if r.status_code == 404:
	continue
	if int(r.headers['Content-Length']) == 0: # 2021-09-06
	continue
	r.raise_for_status()
	with open(FILENAME(day), 'wb') as f:
	f.write(r.content)
	return True
	print("Document not found")
	return False

	def analyse(day: datetime.date):
	""" We are actually only interested in the table in the bottom-right corner.
	Those are actually two tables, or a table on two 'columns'.
	Line 'headers' (city names) are right-justified, and the values are centered.
	On the 2021-11-05 they changed the order of the fields.
	"""
	# y -> name
	cities = {}
	# city name -> [infected, quarantined]
	res = {}
	unmatched = []
	with open(FILENAME(day), "rb") as f:
	# We are only interested in the first page
	page = next(extract_pages(f))
	page_width = page.width
	page_height = page.height
	for element in page:
	if not isinstance(element, LTTextContainer):
	continue
	for line in element:
	if not hasattr(line, 'x1'):
	continue
	if (line.x1 < page_width / 2) or (line.y0 > page_height / 2):
	continue
	print(f"({line.x0:.0f}, {line.y0:.0f}): {line.get_text().strip()}")
	try:
	intVal = locale.atoi(line.get_text())
	if not cities:
	unmatched.append((line.y0, line.height, intVal))
	continue
	city = next(filter(lambda y: abs(y - line.y0) < line.height / 2, cities.keys()))
	if city:
	name = cities[city]
	if res[name][0] is None:
	res[name][0] = intVal
	else:
	res[name][1] = intVal
	except ValueError:
	# Not a number
	name = line.get_text().strip()
	cities[line.y0] = name
	res[name] = [None, None]
	for yVal, heightVal, val in unmatched:
	city = next(filter(lambda y: abs(y - yVal) < heightVal / 2, cities.keys()))
	if city:
	name = cities[city]
	if res[name][0] is None:
	res[name][0] = val
	else:
	res[name][1] = val
	# remove the unmatched artefacts
	return {k: v for k, v in res.items() if v[0] is not None and v[1] is not None}

	def download_all():
	one_day = datetime.timedelta(days=1)
	day = datetime.date.today()
	last = None
	while True:
	if os.path.exists(FILENAME(day)):
	last = day
	break
	# Will break on first three non-available files (29.10.2020)
	if not download_file(day):
	if day != datetime.date.today() and (day - last).days > 3 :
	break

	last = day
	day = day - one_day

	def analyse_all():
	one_day = datetime.timedelta(days=1)
	day = datetime.date.today() + one_day
	skip = 0
	while True:
	day = day - one_day
	print(f"analysing {day}")
	# Will break on first missing file
	dayStr = day.isoformat()
	with open('data/data.json') as f:
	data = json.load(f)
	if dayStr in data:
	continue
	try:
	res = analyse(day)
	except FileNotFoundError:
	skip += 1
	if skip > 3:
	break
	continue
	data[dayStr] = res
	with open('data/data.json', 'wt') as f:
	json.dump(data, f, indent=True)

	download_all()
	analyse_all()

	def validate_data():
	with open('data/census.json') as f:
	CENSUS = json.load(f)

	with open('data/data.json') as f:
	DATA = json.load(f)

	cities = set(CENSUS.keys())

	good = True
	for k, v in DATA.items():
	entries = set(v.keys())
	if cities - entries or entries - cities:
	print (f"{k}: missing: {cities - entries}, too much: {entries - cities}")
	good = False

	return good

	class NoData(Exception):
	pass

	class DailyData(object):
	def __init__(self):
	# date -> amount
	self.data = {}

	def add(self, date, amount):
	self.data[date] = amount

	def get(self, date):
	""" returns the incidence for the day """
	lastWeek = date - datetime.timedelta(days=7)
	if lastWeek not in self.data or date not in self.data:
	raise NoData()
	return self.data[date] - self.data[lastWeek]


	def incidence():
	with open('data/census.json') as f:
	CENSUS = json.load(f)

	with open('data/data.json') as f:
	DATA = json.load(f)

	with open('data/incidence.csv', 'w', newline = '') as f:
	fieldnames = ['day'] + sorted(CENSUS.keys())
	writer = csv.DictWriter(f, fieldnames = fieldnames)

	writer.writeheader()
	# city -> dailydata
	data = {}
	for day in sorted(map(lambda s: datetime.date.fromisoformat(s), DATA.keys())):
	daydata = DATA[day.isoformat()]
	row = {'day': day.isoformat()}
	for city in CENSUS.keys():
	# 0 is amount of infected, 1 is quarantined
	# Add the new value
	data.setdefault(city, DailyData()).add(day, daydata[city][0])
	try:
	row[city] = (data[city].get(day) / CENSUS[city]) * 100_000
	except NoData:
	pass
	if len(row.keys()) > 1: # day
	writer.writerow(row)

	print(row)


	if not validate_data():
	print("Bad, bad, bad ...")
	incidence()