barrucadu

#!/usr/bin/env python
"""
A one-off script to migrate documents from the ES5 metasearch index to ES6.

This is necessary because trying to republish all best bets from search-admin frequently fails due to transient network issues.

Requires two Elasticsearch clients: client5 and client6. These use different versions of the python client.

client5 fetches a page of docs from the old index, and client6 POSTs them to the new index. Simple!
"""

barrucadu

michaelswalker@ec2-integration-blue-search-ip-10-1-4-119:/var/apps/search-api$ ls
michaelswalker@ec2-integration-blue-search-ip-10-1-4-119:/var/apps/search-api$ cd ../search-api
michaelswalker@ec2-integration-blue-search-ip-10-1-4-119:/var/apps/search-api$ ls
bin  config  config.ru  console  doc  Dockerfile  elasticsearch.yml  env.rb  Gemfile  Gemfile.lock  Jenkinsfile  lib  LICENSE.txt  log  Procfile  public  Rakefile  README.md  REVISION  spec  startup.sh  tmp

barrucadu

diff --git a/lib/search/format_migrator.rb b/lib/search/format_migrator.rb
index c2497b63..61d8b997 100644
--- a/lib/search/format_migrator.rb
+++ b/lib/search/format_migrator.rb
@@ -5,13 +5,17 @@ module Search
     end
 
     def call
-      {
-        indices: {

barrucadu

#! /usr/bin/env nix-shell
#! nix-shell -i python3 -p "python3.withPackages (p: [p.requests])"

import os
import requests
from xml.etree import ElementTree


def get_xml(url):
    """Stream some XML and return an iterator over the elements.

barrucadu

2: ens3: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc pfifo_fast state UP group default qlen 1000
    link/ether 96:00:00:20:2c:86 brd ff:ff:ff:ff:ff:ff
    inet 116.203.134.200/32 brd 116.203.134.200 scope global ens3
       valid_lft forever preferred_lft forever
    inet6 2a01:4f8:c2c:2b22::/64 scope global 
       valid_lft forever preferred_lft forever
    inet6 fe80::9400:ff:fe20:2c86/64 scope link 
       valid_lft forever preferred_lft forever

barrucadu

> import qualified Control.Concurrent.Classy as C
> import Control.Monad.ST
> import Control.Monad.Catch.Pure
> resultsSet defaultWay defaultMemType C.supportsBoundThreads
fromList [Right True]
> runST $ runCatchT $ resultsSet defaultWay defaultMemType C.supportsBoundThreads
Right (fromList [Right False])
> resultsSet defaultWay defaultMemType $ C.forkOS (pure ())
fromList [Right 1]
> runST $ runCatchT $ resultsSet defaultWay defaultMemType $ C.forkOS (pure ())

barrucadu

The contents of https://www.gov.uk/api/content/vehicle-tax

barrucadu

    if 'document_type' not in raw:
        raise NoDocumentType()

    document_type = raw['document_type']
    document_type_parser = f'parse_type_{document_type}'

    if document_type_parser not in globals():
        raise UnknownDocumentType(document_type)

    return globals()[document_type_parser](raw)

barrucadu

; treat the remaining balance on the credit card as an asset.

2019-01-01 Got a new credit card
    assets:creditcard                                                  £10000.00
    liabilities:creditcard
2019-02-01 Spending some money on my credit card
    expenses:ducks                                                       £100.00
    assets:creditcard
2019-03-01 Spending some more money on my credit card
    expenses:geese                                                       £100.00

barrucadu

for index in page-traffic metasearch govuk government detailed; do
  for doctype in aaib_report asylum_support_decision best_bet business_finance_support_scheme \
                 cma_case contact countryside_stewardship_grant dfid_research_output drug_safety_update \
                 edition employment_appeal_tribunal_decision employment_tribunal_decision \
                 european_structural_investment_fund export_health_certificate hmrc_manual \
                 hmrc_manual_section international_development_fund maib_report manual manual_section \
                 medical_safety_alert page-traffic policy raib_report residential_property_tribunal_decision \
                 service_manual_guide service_manual_topic service_standard_report tax_tribunal_decision \
                 utaac_decision statutory_instrument; do
    es2_count=`curl http://localhost:9200/$index/$doctype/_count 2>/dev/null | jq .count`