qingdan/preprocess/processDe.py

import pandas as pd
import os
import copy
path = "/Users/xiaopengzhang/Downloads/qingdan/de"
dirs = os.listdir("/Users/xiaopengzhang/Downloads/qingdan/de")
def processDes(dataframe, from_, end, level):
    result = []
    for i  in range(from_, end):
        item = dataframe.iloc[i]
        fbcch = str(item['fbcch'])
        if fbcch == str(level):
            entry = {"id": str(item["ID"]), "label": str(item["zjbt"]), "zjh": str(item["zjh"])}
            next = i + 1
            while next < end and str(dataframe.iloc[next]["fbcch"]) != str(level):
                next = next + 1
            entry["children"] = processDes(dataframe, i + 1, next, level + 1)
            result.append(entry)
    return result
for dir in dirs:
    print(dir)
    files = os.listdir(os.path.join(path, dir))
    def traverse(node, name, collection):
        name.append(node["label"])
        child = node["children"]
        if len(child) == 0:
            name.append(node["zjh"])
            collection.append(copy.deepcopy(name))
            name.pop()
        else:
            for entry in child:
                traverse(entry, name, collection)
        name.pop()

    for file in files:
        if "DingEShu" in file:
            print(file)  
            df = pd.read_csv(os.path.join(path, dir, file))
            df2 = pd.read_csv(os.path.join(path, dir, "JD_DanWeiGJ.csv"))
            newdf = []
            length = len(df)
            tree = processDes(df, 1, length, 2)
            collection = []
            for entry in tree:
                name = []
                traverse(entry, name, collection)
            #print(collection)
            for entry in collection:
                bh = entry[-1]
                filtered = df2[df2["YSDELBH"] == bh]
                if len(filtered) == 0:
                    filtered = df2[df2["YSDELBH"] == int(bh)]
                ##print(filtered)
                filtered["parent"] = " ".join(entry[:-1])
                #print(filtered)
                newdf.append(filtered)
            processed = pd.concat(newdf)
            #print(processed)
            processed.to_csv(os.path.join(path, dir, "DingE_Processed" + dir + ".csv"))
    ##break