Inst2Vec/process_data/shell_com/remove_repete_line.py

19 lines
515 B
Python
Raw Permalink Normal View History

2024-04-11 16:43:57 +08:00
# Python运行shell脚本
import subprocess
import os
from my_utils import multi_thread
def run_shell(file_num):
com_line = f'cat ./neg_txt/inst.{file_num}.neg.txt | sort -n | uniq > ./neg_clean/inst.{file_num}.neg.txt.clean'
p = subprocess.Popen(com_line, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = p.communicate()
return stdout, stderr
if __name__ == '__main__':
os.chdir('../../dataset/all')
result = multi_thread(run_shell, range(os.cpu_count()))