mdouze

mdouze

path: /private/home/matthijs/faiss_versions/faiss_for_improvements/build/faiss/python/build/lib/faiss/contrib/datasets.py
args: Namespace(M0=-1, add_bs=1000000, autotune_max=['nprobe:3000'], autotune_range=[], by_residual=-1, clustering_niter=-1, compute_gt=False, db='deep1B', get_centroids_from='/checkpoint/matthijs/bench_all_ivf/precomputed_clusters/clustering.dbdeep1M.IVF4194304.faissindex', indexfile='/checkpoint/matthijs/bench_all_ivf/indexes/autotune.dbdeep1B.OPQ32_128_IVF4194304_HNSW32_PQ32.faissindex', indexkey='OPQ32_128,IVF4194304_HNSW32,PQ32', inter=False, k=100, maxtrain=0, min_test_duration=3.0, n_autotune=500, no_precomputed_tables=False, searchparams=['autotune'], searchthreads=32, train_on_gpu=False)
nb processors 80
model name	: Intel(R) Xeon(R) Gold 6138 CPU @ 2.00GHz
load data deep1B
dataset in dimension 96, with metric L2, size: Q 10000 B 1000000000 T 358480000
reading /checkpoint/matthijs/bench_all_ivf/indexes/autotune.dbdeep1B.OPQ32_128_IVF4194304_HNSW32_PQ32.faissindex
index size on dis

mdouze

parameters                                   R@1   R@10  R@100  time(ms/q)   nb distances #runs
                                         0.9902 0.9971 0.9971      0.00732      741210288    42
k_factor_rf=1,nprobe=2                   0.6133 0.6174 0.6174      0.00200       28321230    150
k_factor_rf=1,nprobe=4                   0.7632 0.7674 0.7674      0.00242       55260251    124
k_factor_rf=1,nprobe=8                   0.8749 0.8811 0.8811      0.00274      106710942    110
k_factor_rf=1,nprobe=16                  0.9472 0.9543 0.9543      0.00361      204804549    84
k_factor_rf=1,nprobe=32                  0.9797 0.9858 0.9858      0.00499      391006867    61
k_factor_rf=1,nprobe=64                  0.9902 0.9971 0.9971      0.00712      741210288    43
k_factor_rf=4,nprobe=64                  0.9905 0.9974 0.9974      0.01139      741210288    27
k_factor_rf=1,nprobe=128                 0.9917 0.9995 0.9995      0.01109     1400811424    28

mdouze

mdouze

mdouze

(faiss_latest_cmake) matthijs@devfair0144:~/faiss_versions/faiss_latest_cmake$ make -j -C build demo_sift1M
make: Entering directory '/private/home/matthijs/faiss_versions/faiss_latest_cmake/build'
make[1]: Entering directory '/private/home/matthijs/faiss_versions/faiss_latest_cmake/build'
make[2]: Entering directory '/private/home/matthijs/faiss_versions/faiss_latest_cmake/build'
make[3]: Entering directory '/private/home/matthijs/faiss_versions/faiss_latest_cmake/build'
make[3]: Leaving directory '/private/home/matthijs/faiss_versions/faiss_latest_cmake/build'
make[3]: Entering directory '/private/home/matthijs/faiss_versions/faiss_latest_cmake/build'
[ 16%] Building CXX object faiss/CMakeFiles/faiss.dir/IndexHNSW.cpp.o
[ 35%] Building CXX object faiss/CMakeFiles/faiss.dir/IndexIVFFlat.cpp.o
[ 42%] Building CXX object faiss/CMakeFiles/faiss.dir/IndexIVF.cpp.o

mdouze

Scanning dependencies of target example_c
make[2]: Leaving directory '/private/home/matthijs/faiss_versions/faiss_c_interface/build'
make  -f c_api/CMakeFiles/example_c.dir/build.make c_api/CMakeFiles/example_c.dir/build
make[2]: Entering directory '/private/home/matthijs/faiss_versions/faiss_c_interface/build'
[ 97%] Building C object c_api/CMakeFiles/example_c.dir/example_c.c.o
cd /private/home/matthijs/faiss_versions/faiss_c_interface/build/c_api && /usr/bin/cc   -O3 -DNDEBUG -o CMakeFiles/example_c.dir/example_c.c.o -c /private/home/matthijs/faiss_versions/faiss_c_interface/c_api/example_c.c
[ 98%] Linking CXX executable example_c
cd /private/home/matthijs/faiss_versions/faiss_c_interface/build/c_api && /private/home/matthijs/faiss_versions/faiss_20200817/current_cmake/bin/cmake -E cmake_link_script CMakeFiles/example_c.dir/link.txt --verbose=1
/usr/bin/c++ -O3 -DNDEBUG CMakeFiles/example_c.dir/example_c.c.o -o example_c  libfaiss_c.a ../faiss/libfaiss_avx2.a /usr/lib/gcc/x86_64-linux-gnu/7/libgomp.so -lp

mdouze

(faiss_1.6.3) matthijs@devfair0144:~/faiss_versions/issue_1350$ python test1.py
i=0 RSS=56520 kB
i=200 RSS=67104 kB
i=400 RSS=67220 kB
i=600 RSS=68368 kB
i=800 RSS=68800 kB
i=1000 RSS=68800 kB
i=1200 RSS=68800 kB
i=1400 RSS=68800 kB
i=1600 RSS=68868 kB

mdouze

import faiss

import numpy as np

tab = {}

for i in range(10**6):
   print(f'i={i} RSS={faiss.get_mem_usage_kb()} kB', end='\n' if i % 200 == 0 else '\r', flush=True)
   quant = faiss.IndexFlatL2(32)
   index = faiss.IndexIVFPQ(quant, 32, 100, 4, 4)

mdouze