Jongbhin

    with open(input_file) as ifd:
        csv_reader = csv.reader(ifd, delimiter='\t', quoting=csv.QUOTE_NONE)

        token_dict = {}
        for counter, row in enumerate(csv_reader):
            prd_no = row[0]
            token_dict[prd_no] = row[1:]

    with open(output_file, mode='w') as wfd:
        pickle.dump(token_dict, wfd, protocol=pickle.HIGHEST_PROTOCOL)

Jongbhin

generate rsa key

ssh-keygen

add to server

ssh-copy-id -i ~/.ssh/id_rsa.pub 1100378@host

Jongbhin

# -*- coding: utf-8 -*-
from __future__ import print_function

import os
import csv

output_dir = ''
input_file = os.path.join(output_dir, 'input0000')
output_file = os.path.join(output_dir, 'input0000.tsv')

Jongbhin

Shell script

#!/bin/bash
for i in $(seq -f "%04g" 0000 0249)
do
    echo "=========================================================================================================="
    input_file="input$i"
    echo $input_file
    /app/jdk/bin/java -Xmx2048m -jar -Dtarget=$input_file cleanscoring-batch-0.0.1.jar cleanScoringJob >> ./logs/clean_scoring_all_product.log 2>&1
done

Jongbhin

Frequently used cmommand

conda env list
conda config --envs
conda config --set auto_activate_base false
conda remove --name myenv --all
conda create -n py27 python=2.7 anaconda
conda env remove -n ENV_NAME

Jongbhin

# for ann in dic
cd $CONDA_PREFIX
mkdir -p ./etc/conda/activate.d
mkdir -p ./etc/conda/deactivate.d
touch ./etc/conda/activate.d/env_vars.sh
touch ./etc/conda/deactivate.d/env_vars.sh

# set alias for pip
cat >> ./etc/conda/activate.d/env_vars.sh << EOF
alias pip3='/app/recsy11/.conda/envs/ann/bin/pip3'

Jongbhin

crontab backup

00 00 * * * crontab -l &> /app/puser/crontab_list ~

Category Mapping

• CTGRYi-11stbat02 : 172.21.16.82 v
• IMGPRCSb-ctgr01 : 172.28.4.65 v
• IMGPRCSb-ctgr02 : 172.28.4.66 v
• IMGPRCSb-ctgr03 : 172.28.4.67

Clean Scoring

Jongbhin

To check nvidia driver

modinfo nvidia

To check cuda version

cat /usr/local/cuda/version.txt

Jongbhin

#!/usr/bin/env python3

import pyvips
import cv2
from PIL import Image

import timeit
import numpy as np
import time
import cProfile

Jongbhin

split_cmd = "split -n l/{} -d -a 2 {} meta_input".format(file_num, file_path)


nohup /app/puser/catalog_preprocessor/src/run.sh today 1002,1003,153118,1001,1004,153106,1006,1007,153108,1531201,1005,153111,153112 &> /app/puser/catalog_preprocessor/log/run.log.`date +"\%Y\%m\%d"` &

신규생성된 상품은 일주일간 추가매칭에서 제외한다.
신규생성이 존재하면 추가매칭은 제거한다.

11->12일 증분 결과