Arne Sund arnesund

arnesund / fetch_netatmo.py

Last active September 3, 2023 13:40

Fetch Netatmo Weather Station measurements and store in InfluxDB

	#!/usr/bin/env python
	import os
	import sys
	import json
	import time
	import requests

	# Get your client ID and secret by creating an App at https://dev.netatmo.com/
	NETATMO_CLIENT_ID = ""
	NETATMO_CLIENT_SECRET = ""

arnesund / hashtag_word_count.py

Created July 13, 2015 13:55

Word count of Twitter hashtags using Apache Spark

	# Count the number of occurrences for each hashtag,
	# by first extracting the hashtag and lowercasing it,
	# then do a standard word count with map and reduceByKey
	countsRDD = (filteredTweetsRDD
	.flatMap(lambda tweet: [hashtag['text'].lower() for hashtag in tweet['entities']['hashtags']])
	.map(lambda tag: (tag, 1))
	.reduceByKey(lambda a, b: a + b)
	)

	# Get the most used hashtags (order countsRDD descending by count)

arnesund / load_and_filter_tweets.py

Created July 13, 2015 12:34

Load tweets into Spark and filter

	# Extract tweets from MongoDB
	allTweets = []
	for doc in db.tweets.find():
	allTweets.append(doc['tweet'])

	# Load tweets into Spark for analysis
	allTweetsRDD = sc.parallelize(allTweets, 8)

	# Set up filter to only get tweets from the last week
	DAYS_LIMIT=7

arnesund / get_tweets.py

Last active January 30, 2017 23:41

Fetch tweets from Twitter using Python module Tweepy

	# Get details about own user
	me = api.me()
	friends = api.friends_ids(me.id)

	# Initialize data structure
	tweets = {}

	# Fetch lists recent tweets for each of the user IDs in the list 'friends'
	for user in friends:
	# Only query Twitter for data not already cached

arnesund / cloudinit.conf

Created February 5, 2015 13:10

Simple Cloud-init Config Example for Apache