Scraping http://alkitab.mobi/

index.js

const client = new ApolloClient({
  networkInterface: createNetworkInterface({
    uri: "https://n84vv4q37.lp.gql.zone/graphql"
  })
});

// Define your models and their properties
const PassageSchema = {
  name: "Passage",
  primaryKey: "id",
  properties: {
    id: "string",
    content: "string",
    book: "string",
    chapter: "int",
    verse: "int",
    type: "string",
    order: "int"
  }
};

app.get("/", (req, res) => {
  Realm.open({ schema: [PassageSchema] }).then(realm => {
    books.forEach(book => {
      for (var index = 1; index <= book.total; index++) {
        let passages = realm.objects("Passage");
        let filteredPassages = passages.filtered(`book = "${book.value}" AND chapter = "${index}"`);

        if (Object.keys(filteredPassages).length == 0) {
          client
            .query({
              query: gql`
                query AlkitabAPI($chapter: Int!, $book: String!) {
                  passages(version: nkjv, book: $book, chapter: $chapter) {
                    verses {
                      content
                      verse
                      type
                      order
                    }
                    chapter
                    book
                    version
                  }
                }
              `,
              variables: {
                chapter: index,
                book: book.value
              }
            })
            .then(data => {
              // res.json(data);
              const { passages } = data.data;
              console.log(`Scraping ${passages.book} ${passages.chapter}`);
              passages.verses.forEach(verse => {
                realm.write(() => {
                  const id = `${passages.book}-${passages.chapter}-${verse.verse}-${verse.type[0]}`;
                  const passage = realm.create(
                    "Passage",
                    {
                      id: id,
                      content: verse.content,
                      book: passages.book,
                      chapter: passages.chapter,
                      verse: verse.verse,
                      order: verse.order,
                      type: verse.type[0]
                    },
                    true
                  );
                });
              });
            })
            .catch(error => console.error(error));
        } else {
          console.log(`Content already scraped ${book.value} ${index}`);
        }
      }
    });
  });
  res.json({ start: true });
});