Last active
February 10, 2023 14:02
-
-
Save marcosbarker/ec83e93589af46ea5e12c2422c97df3c to your computer and use it in GitHub Desktop.
sw2d_kf_rna_notebook.ipynb
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| { | |
| "cells": [ | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "view-in-github", | |
| "colab_type": "text" | |
| }, | |
| "source": [ | |
| "<a href=\"https://colab.research.google.com/gist/marcosbarker/ec83e93589af46ea5e12c2422c97df3c/-sw2d_kf_rna_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "wPKWPrjVsJl7" | |
| }, | |
| "source": [ | |
| "# Exemplo de como obter a RNA para emular FK no problema de Shalow Water 2D" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "colab": { | |
| "background_save": true | |
| }, | |
| "id": "aE9D44WssL9b" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "!apt install octave" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "pXcpJb5tsphv" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "!pip install oct2py" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "sJqdi1Srts35" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "%load_ext oct2py.ipython" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "e379KNMcsJl-" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Baixa o código KFS2d_rna (shallow water assimilado por FK e RNA) a partir do repositório espelho do github\n", | |
| "!git clone https://github.com/robertopsouto/kfs2d_rna_mirror.git" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "4ZqDc4TGsJl_" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Entra no diretório principal do código de shalow water\n", | |
| "%cd kfs2d_rna_mirror" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "3uwc7H9TsJl_" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Compilando o código\n", | |
| "!make" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "zJCr_ZWRsJmA" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Roda o shell script que chama o executável compilado na célula anterior\n", | |
| "# Usa filtro de Kalman para fazer a assimilação de dados a cada 10 passos no tempo.\n", | |
| "# O ruído no dado de observação é de 10%\n", | |
| "# Uso:\n", | |
| "# ./run-KFS2d.sh assimType gridX gridY timeStep freqObsT freqObsX freqObsY percNoise neuronNumber\n", | |
| " \n", | |
| "!./run-KFS2d.sh 1 10 10 200 10 2 2 0.1 10 1" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "W-OPPEyrsJmA" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Indo para o diretório com o dados que serão usados para treinar a RNA, por meio do MPCA\n", | |
| "%cd output/training/" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "_vcN1S4zsJmB" | |
| }, | |
| "source": [ | |
| "## Script `Octave` que normaliza os valores do modelo, da observação e da análise (obtida com a assimilação de dados). São gerados os arquivos de treinamento (`x.txt` e `y.txt`) de validação cruzada (`x_valid.txt` e `y_valid.txt`) e de generalização (`x_gen.txt` e `y_gen.txt`), que serão os dados de entrada para o `MPCA-ANN`." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "0UYsCtATyYv5" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "%%octave\n", | |
| "qM = load('qModelExpA.out');\n", | |
| "qO = load('qObservExpA.out');\n", | |
| "qA = load('qAnalysisExpA.out');\n", | |
| " \n", | |
| "valNormInf=-1.0;\n", | |
| "valNormSup=+1.0;\n", | |
| " \n", | |
| "qM_norm=(max(qM)*valNormInf-min(qM)*valNormSup+qM*(valNormSup-valNormInf))/(max(qM)-min(qM));\n", | |
| "qO_norm=(max(qO)*valNormInf-min(qO)*valNormSup+qO*(valNormSup-valNormInf))/(max(qO)-min(qO));\n", | |
| "qA_norm=(max(qA)*valNormInf-min(qA)*valNormSup+qA*(valNormSup-valNormInf))/(max(qA)-min(qA));\n", | |
| " \n", | |
| "[l,n]=size(qM);\n", | |
| "pontos_por_ciclo=25;\n", | |
| " \n", | |
| "parc1 = 10*pontos_por_ciclo; \n", | |
| "parc2 = 2*pontos_por_ciclo; \n", | |
| "parc3 = 4*pontos_por_ciclo; \n", | |
| " \n", | |
| "train_range=parc1;\n", | |
| "valid_init=train_range+1;\n", | |
| "valid_end=train_range+parc2;\n", | |
| "gen_init=valid_end+1;\n", | |
| "gen_end=valid_end+parc3;;\n", | |
| " \n", | |
| "fileID = fopen('x.txt','w');\n", | |
| "fprintf(fileID,' %8.5f',qM_norm(1:train_range));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);\n", | |
| "fileID = fopen('x.txt','a');\n", | |
| "fprintf(fileID,' %8.5f',qO_norm(1:train_range));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);\n", | |
| "fileID = fopen('y.txt','w');\n", | |
| "fprintf(fileID,' %8.5f',qA_norm(1:train_range));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);\n", | |
| " \n", | |
| "fileID = fopen('x_valid.txt','w');\n", | |
| "fprintf(fileID,' %8.5f',qM_norm(valid_init:valid_end));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);\n", | |
| "fileID = fopen('x_valid.txt','a');\n", | |
| "fprintf(fileID,' %8.5f',qO_norm(valid_init:valid_end));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);\n", | |
| "fileID = fopen('y_valid.txt','w');\n", | |
| "fprintf(fileID,' %8.5f',qA_norm(valid_init:valid_end));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);\n", | |
| " \n", | |
| "fileID = fopen('x_gen.txt','w');\n", | |
| "fprintf(fileID,' %8.5f',qM_norm(gen_init:gen_end));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);\n", | |
| "fileID = fopen('x_gen.txt','a');\n", | |
| "fprintf(fileID,' %8.5f',qO_norm(gen_init:gen_end));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);\n", | |
| "fileID = fopen('y_gen.txt','w');\n", | |
| "fprintf(fileID,' %8.5f',qA_norm(gen_init:gen_end));\n", | |
| "fprintf(fileID,'\\n');\n", | |
| "fclose(fileID);" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "d6vrw80TzpLk" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "%octave_pull parc1\n", | |
| "%octave_pull parc2\n", | |
| "%octave_pull parc3" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "ntQZUrlbsJmC" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "parc1" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "Ub0G2IyAsJmD" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "parc2" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "lmgQkXeIsJmD" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "parc3" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "BuHJUAx6sJmD" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "%cd ../../../" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "2h2nPEDIsJmE" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Baixa o código do mpac-ann do repositório github\n", | |
| "!git clone https://github.com/scsr-inpe/mpca-ann.git" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "zGq7KdyWsJmE" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Entra no diretório principal do código\n", | |
| "%cd mpca-ann" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "i-0LLqMjsJmE" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Alterna para um commit específico\n", | |
| "!git checkout a52ad36" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "QOsvyL6rsJmE" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Remove os arquivos pré-compilados que são baixados\n", | |
| "!make clean" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "fY-sr-yLsJmF" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Compila o código \n", | |
| "!make" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "PemHTZoysJmF" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Copia os arquivos de treinamento, de validação cruzada e de generalização gerados anteriormente no k, para o diretório 'data' do mpca-ann\n", | |
| "!cp ../kfs2d_rna_mirror/output/training/*.txt data/" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "64RjcabysJmF" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Entra no diretório config, para editar os arquivos de configuração do mpca-ann\n", | |
| "%cd config/" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "KucMFahCsJmF" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Edita o arquivo original configuration.ini tal que:\n", | |
| "# NCLASSES seja igual ao número de pontos de treinamento em x.txt e y.txt (dado pela variável parc1) \n", | |
| "# NCLASSESVALIDATION seja igual ao número de pontos de validação em x_valid.txt e y_valid.txt (dado pela variável parc2)\n", | |
| "# UPPER_HIDDEN_LAYERS=1 seja igual a LOW_HIDDEN_LAYERS=1, para forçar que a arquitetura da rede tenha somente uma camada\n", | |
| "# LOWER_FIRST_HIDDEN_LAYER=UPPER_FIRST_HIDDEN_LAYER=10, para forçar que esta camana tenha exatamente 10 neurônios\n", | |
| "# LOWER_ACTIVATION_FUNCTION=UPPER_ACTIVATION_FUNCTION=2, para forçar que se use com função de ativação a tangente hiperbólica\n", | |
| " \n", | |
| "!sed -i -e 's/NCLASSES=2000/NCLASSES=250/g' \\\n", | |
| " -e 's/NCLASSESVALIDATION=600/NCLASSESVALIDATION=50/g' \\\n", | |
| " -e 's/UPPER_HIDDEN_LAYERS=2/UPPER_HIDDEN_LAYERS=1/g' \\\n", | |
| " -e 's/LOWER_FIRST_HIDDEN_LAYER=5/LOWER_FIRST_HIDDEN_LAYER=10/g' \\\n", | |
| " -e 's/UPPER_FIRST_HIDDEN_LAYER=25/UPPER_FIRST_HIDDEN_LAYER=10/g' \\\n", | |
| " -e 's/LOWER_ACTIVATION_FUNCTION=1/LOWER_ACTIVATION_FUNCTION=2/g' \\\n", | |
| " -e 's/UPPER_ACTIVATION_FUNCTION=3/UPPER_ACTIVATION_FUNCTION=2/g' \\\n", | |
| " configuration.ini" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "7UC2uFdjsJmG" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Edita o arquivo original annConfig.ini, \n", | |
| "# para que o número de classes seja igual ao número de pontos generalização em x_gen.txt e y_gen.txt\n", | |
| " \n", | |
| "!sed -i -e 's/600/100/g' annConfig.in" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "Zt6nJA_AsJmG" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Retorna para o diretório do script MPCA-ANN\n", | |
| " \n", | |
| "%cd .." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "G2HKTSEMsJmH" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Cria o diretório output para armazenar o resultado do mpca-ann\n", | |
| " \n", | |
| "!mkdir output" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "X30i6IqnsJmH" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Roda e script com 1 Experimento, usando 1 Processo MPI\n", | |
| " \n", | |
| "!./annMPCA 1" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "MD-0HGrVsJmH" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Roda esta etapa para obter a solução que possui o menor valor da função objetivo com relação aos dados de generalização\n", | |
| " \n", | |
| "!./annMLP 1 1" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "_6wgtg6MsJmH" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Copia o arquivo ann#.best, relativo ao experimento que obteve no menor erro na etapa de generalização, para o diretório data\n", | |
| " \n", | |
| "!cp output/ann1.best ../kfs2d_rna_mirror/data/ann.best" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "pBmbAlGNsJmI" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Entra no diretório data\n", | |
| " \n", | |
| "%cd ../kfs2d_rna_mirror/data/" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "_a7HzHrosJmI" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Roda o shell script que extrai a informação dos pesos e vies do arquivo ann.best\n", | |
| "# São gerados 4 arquivos: wqcsExpA.dat wqcoExpA.dat bqcsExpA.dat bqcoExpA.dat\n", | |
| "# Estes 4 arquivos serão lidos pelo kfs2d, para emular o FK por RNA \n", | |
| " \n", | |
| "!./pesos_vies.sh ann.best" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "3-2qdHYhsJmI" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "%cd ../" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "Nly3OjCWsJmI" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Roda o shell script que chama o executável\n", | |
| "# Usa RNA para emular FK na assimilação de dados, a cada 10 passos no tempo.\n", | |
| "# Uso:\n", | |
| "# ./run-KFS2d.sh assimType gridX gridY timeStep freqObsT freqObsX freqObsY percNoise neuronNumber numthreads\n", | |
| " \n", | |
| "!./run-KFS2d.sh 2 10 10 200 10 2 2 0.1 10 1 1" | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "1-vvN8EKsJmJ" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "# Entra no diretório onde o resultado da análise obtido com a assimilação por RNA está salvo\n", | |
| " \n", | |
| "%cd output/full/" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "4l2tyUk8sJmJ" | |
| }, | |
| "source": [ | |
| "## Script `Octave` que gera figura comparando as curvas do modelo (verdade), da análise obtida por assimilação com filtro de Kalman (FK), e também da análise obtida por assimilação com rede neural emulando filtro de Kalman (RNA)." | |
| ] | |
| }, | |
| { | |
| "cell_type": "code", | |
| "execution_count": null, | |
| "metadata": { | |
| "id": "i-MKOrGqsJmJ" | |
| }, | |
| "outputs": [], | |
| "source": [ | |
| "%%octave\n", | |
| " \n", | |
| "qm = load('qModelExpA.out');\n", | |
| "qa = load('qAnalysisExpA.out');\n", | |
| " \n", | |
| "qar = load('qAnalysisExpA_RNA.out');\n", | |
| " \n", | |
| "ni = 5;\n", | |
| "nj = ni;\n", | |
| "nk = 200;\n", | |
| " \n", | |
| "ninj = ni*nj;\n", | |
| "p = 2; % ponto\n", | |
| " \n", | |
| "qm3D = reshape(qm,ni,nj,nk);\n", | |
| "qpm10 = qm3D(p,p,:);\n", | |
| "qpm10v = qpm10(:);\n", | |
| " \n", | |
| "qafk3D = reshape(qa,ni,nj,nk);\n", | |
| "qpa10 = qafk3D(p,p,:);\n", | |
| "qpa10v = qpa10(:);\n", | |
| " \n", | |
| "qr3D = reshape(qar,ni,nj,nk);\n", | |
| "qpr10 = qr3D(p,p,:);\n", | |
| "qpr10v = qpr10(:);\n", | |
| " \n", | |
| "figure(1)\n", | |
| "plot(qpm10v,'b','linewidth',1.5); hold on;\n", | |
| "plot(qpa10v,'r','linewidth',1.5);\n", | |
| "plot(qpr10v,'g','linewidth',1.5);\n", | |
| "title('variavel q'); grid on;\n", | |
| "xlabel('tempo');\n", | |
| "%ylabel('q');\n", | |
| "legend('verdade','FK','RNA')\n", | |
| "axis([1 nk -60 80])\n", | |
| " \n", | |
| "print -dpng -r240 variavelqExpA.png;" | |
| ] | |
| }, | |
| { | |
| "cell_type": "markdown", | |
| "metadata": { | |
| "id": "8Wvrs2_fsJmL" | |
| }, | |
| "source": [ | |
| "" | |
| ] | |
| } | |
| ], | |
| "metadata": { | |
| "colab": { | |
| "provenance": [], | |
| "name": "sw2d_kf_rna_notebook.ipynb", | |
| "include_colab_link": true | |
| }, | |
| "kernelspec": { | |
| "display_name": "Python 3", | |
| "name": "python3" | |
| }, | |
| "language_info": { | |
| "name": "python" | |
| }, | |
| "sos": { | |
| "kernels": [ | |
| [ | |
| "Octave", | |
| "octave", | |
| "Octave", | |
| "#dff8fb", | |
| "" | |
| ], | |
| [ | |
| "Python 3", | |
| "python3", | |
| "python3", | |
| "", | |
| { | |
| "name": "ipython", | |
| "version": 3 | |
| } | |
| ], | |
| [ | |
| "SoS", | |
| "sos", | |
| "", | |
| "", | |
| "sos" | |
| ] | |
| ], | |
| "panel": { | |
| "displayed": false, | |
| "height": 0 | |
| }, | |
| "version": "0.22.3" | |
| } | |
| }, | |
| "nbformat": 4, | |
| "nbformat_minor": 0 | |
| } |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment