aitoehigie

import os
from os.path import dirname, join
from flask import Flask
from dotenv import load_dotenv
#import featurerequests.views (I commented this line out because importing the views file here, has on iine three in the views
# file a line that also calls the app object, but the app object at the stage doesnt exist hence the error. This is called
# a "CIRCULAR IMPORT"
# by moving it to the bottom, after the app = Flask(__name__) declaration, you program runs now.

aitoehigie

import sys
from django.conf import settings
settings.configure(
DEBUG=True,
SECRET_KEY='thisisthesecretkey',
ROOT_URLCONF=__name__,
MIDDLEWARE_CLASSES=(
'django.middleware.common.CommonMiddleware',
'django.middleware.csrf.CsrfViewMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',

aitoehigie

from flask import Flask, jsonify
import os, json
app = Flask(__name__)

@app.route("/nairaland/api/v1.0/homepage", methods=["GET"])
def homepage():
    with app.open_resource("static/data/results.json") as f:
        data = json.load(f)
    return jsonify(data)

aitoehigie

[
{"timestamp": ["2017-05-20 08:43:10"], "title": ["Retired Generals Warn Military To Stay Out Of Politics"]},
{"timestamp": ["2017-05-20 08:43:10"], "title": ["Throwback Photo", "Of", "Oyo First Lady, Mrs Florence Ajimobi", "In", "1966"]},
{"timestamp": ["2017-05-20 08:43:10"], "title": ["Why Does Naira Appreciate Whenever President Buhari Travels?"]},
{"timestamp": ["2017-05-20 08:43:10"], "title": ["Senator Dino Melaye\u2019s Father Prays On His Anti-Corruption Book", "(", "Photo", ")"]},
{"timestamp": ["2017-05-20 08:43:10"], "title": ["\"Why Northern Leaders Worked Against Jonathan\u2019s Government\" \u2013 Paul Unongo"]},
{"timestamp": ["2017-05-20 08:43:10"], "title": ["Nollywood Actor, Jim Iyke Visits Nnamdi Kanu", "(", "Photos", ")"]},
{"timestamp": ["2017-05-20 08:43:10"], "title": ["Governor Fayose", "Vs", "Governor Ambode: Comparison", "By", "APC London", "(", "Photos", ")"]},
{"timestamp": ["2017-05-20 08:43:10"], "title": ["Peter Obi Spotted Walking Home", "On", "Lagos Bridge After Church Servi

aitoehigie

scrapy crawl nairaland -o results.json

#This command will crawl the nairaland home page and save the results of the news items in a file names 'results.json'

aitoehigie

# -*- coding: utf-8 -*-
import scrapy
from scrapy.selector import Selector
from scrapy.loader import ItemLoader
from scrapy.loader.processors import MapCompose, Join
from nairalandapi.items import NairalandapiItem
import datetime

class NairalandapiSpider(scrapy.Spider):
    name = "nairaland"

aitoehigie

import scrapy

#this items file is just going to hold 3 items.

title = scrapy.Field()
link = scrapy.Field()
#house keeping field
timestamp = scrapy.Field()

aitoehigie

# -*- coding: utf-8 -*-
import scrapy


class NairalandSpider(scrapy.Spider):
    name = "nairaland"
    allowed_domains = ["nairaland.com"]
    start_urls = ['http://nairaland.com/']

    def parse(self, response):

aitoehigie

scrapy genspider nairaland nairaland.com

aitoehigie

scrapy startproject nairalandapi