大疫情数据查重Python

摘要：如果文件太大的话比较吃电脑的配置思路及代码用自带的做成更加普遍好用。

每个月要对疫情卡片进行查重，而服务器只提供一段时间的查重功能，无法对指定时间的卡片进行查重！

用Excel查重，效率太低。如果文件太大的话比较吃电脑的配置!

用Python自带的TK做成GUI更加普遍好用。

用pandas包来实现数据的查重功能，具体见代码注释！源代码见文后：

默认为名字拼音+疾病名称 与 身份证号+疾病名称的查重的交集（身份证查重的优先级高于名字拼音），如勾选了 性别 现住址国标的话，则判断重卡的依据就是 姓名拼音+性别+现住址+疾病名称为相同则判定为重卡！！其它类似！

因为为python打包而来，所以双击之后请稍等（解析速度较慢）！

如果杀毒软件提示，请点击允许允许！

如果双击之后出现找不到什么动态库，如下界面的时候，请安装前往https://www.microsoft.com/zh-... 下载安装VC2015之后在运行

如果安装以上下载文件出错时，请用杀毒软件下载系统更新补丁后重试！

如果在使用过程中有什么疑问的或好的建议的，可以发送邮件到ztwenxing@dingtalk.com（有时间的会回复）

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from pinyin import pinyin
from tkinter import Tk, filedialog, Canvas, messagebox, StringVar, IntVar
from tkinter import Label, Entry, Button, Checkbutton
# from PIL import ImageTk, Image
from pandas import DataFrame, read_csv, Series, to_datetime

##设置窗口界面
window = Tk()
# 设置标题
window.title("大疫情查重用")
# 设置窗口大小
window.geometry("700x395")

# frame=Frame(window)
# frame.pack()
# 设置背景图片
canvas = Canvas(window, width=700, height=395, bd=0, highlightthickness=0)
# imgpath = "giphy.gif"

# 设置背景图片
# img = Image.open("C:/Users/Administrator/chachong/ztcdc3.png")
# photo = ImageTk.PhotoImage(image=img)
width = 700
height = 395

# 画布设置数值为中心点的数值
# canvas.create_image(width/2, height/2, image=photo)
canvas.create_image(width / 2, height / 2)
canvas.pack()
# 设置标签1
##设置输入界面
#
label_text = Label(window, text="此小程序主要用大疫情网络的疫情卡片查重！！！")
# label_text.grid(row=0,column= 1)


# 设置高度的等分比例
height_x = height / (height / 39.5)
col_num = 1
canvas.create_window(width * (9 / 18), height_x * col_num, window=label_text)
col_num = 2


def choiceFileCallBack():
    # 选择文件
    filenames = filedialog.askopenfilenames(filetypes=[("csv文件", "*.csv")])
    # filenames = filedialog.askopenfilenames()
    if len(filenames) != 0:
        if str(u"csv") in filenames[0]:
            en_text.set("")
            string_filename = ""
            string_filename = str(filenames[0])
            # 设置Lb1的属性
            en_text.set(string_filename)
            # 设置bt3的激活属性
            button3.configure(state="active")
        else:
            en_text.set("")
            messagebox.showinfo("请选择csv文件格式", "未选择csv格式文件，请重新选择！！")
            button3.configure(state="disabled")
    else:
        en_text.set("")
        messagebox.showinfo("未选择", "未选择文件,请选择")
        # 设置bt3的激活属性
        button3.configure(state="disabled")


label1 = Label(window, text="需要读取的文件路径：", )
canvas.create_window(width * (2 / 18), height_x * col_num, width=170, window=label1)
en_text = StringVar()  # 绑定listbox的列表值
en_text.set((""))
entry = Entry(window, textvariable=en_text)
canvas.create_window(width * (9 / 18), height_x * col_num, width=360, window=entry)

button1 = Button(window, text="选择", command=choiceFileCallBack)
canvas.create_window(width * (16 / 18), height_x * col_num, width=80, height=30, window=button1)


# 选择保存路径
def choiceSaveCallBack():
    # 选择文件

    filename = filedialog.asksaveasfilename(filetypes=[("csv文件", "*.csv")])

    if filename != "":
        ##判断是否为csv格式的文件
        en1_text.set("")
        string_filename = ""
        string_filename = str(filename) + ".csv"
        # 设置Lb1的属性
        en1_text.set(string_filename)
        # 设置bt3的激活属性
        button3.configure(state="active")

    else:
        en1_text.set("")
        # 设置bt3的激活属性
        messagebox.showinfo("未选择", "未选择保存路径请选择！")
        button3.configure(state="disabled")




col_num = 3
label2 = Label(window, text="查重的保存路径及文件名：", )
canvas.create_window(width * (2 / 18), height_x * col_num, width=170, window=label2)
en1_text = StringVar()  # 绑定listbox的列表值
en1_text.set((""))
entry1 = Entry(window, textvariable=en1_text)
canvas.create_window(width * (9 / 18), height_x * col_num, width=360, window=entry1)
button2 = Button(window, text="选择", command=choiceSaveCallBack)
canvas.create_window(width * (16 / 18), height_x * col_num, width=80, height=30, window=button2)

###设置逻辑层
# 点击OK按钮的函数
def hellook():
    # 设置查重list
    name_lists = ("患者姓名", "有效证件号", "性别", "联系电话", "现住地址国标", "疾病名称", "病例分类", "病例分类2")
    check_lists = [CheckVar1.get(), CheckVar2.get(), CheckVar3.get(), CheckVar4.get(), CheckVar5.get(), CheckVar6.get(),
                   CheckVar7.get(), CheckVar8.get()]
    check_list_pd = DataFrame({"name_lists": name_lists, "check_lists": check_lists})
    check_list = check_list_pd[check_list_pd["check_lists"] == 1]["name_lists"]
    check_list = check_list.values.tolist()
    print(check_list)
    try:
        rc_data = read_csv(filepath_or_buffer=entry.get(), encoding="GB18030")
    except:
        open_error = messagebox.showinfo(title="unfortunately ", message="打开文件出错，请检查！")
        messagebox.showinfo(title="unfortunately ", message="打开文件出错，请检查！")
        print(open_error)

    ### 拼音转换函数
    def hanzi2pinyin(sr, *args, **kwargs):
        list = []
        for i in sr:
            list.append(pinyin.get(i, format="strip", delimiter=""))
        return list

    ##重卡生成函数

    def shengcheng_chongka(data, checklist, aeslist="报告卡录入时间"):
        # data为查重的列,chcklist为查重列的合并list
        # 合并对应的列
        colwx = DataFrame(Series([""] * data.__len__()).str.cat(data[checklist], na_rep="_"))
        colwx = colwx.rename(columns={0: "chachong"})
        # 生成查重的数据格式
        chachong_data_sf = data.reset_index(drop=True)
        chachong_data_sf = chachong_data_sf.merge(colwx, left_index=True, right_index=True)
        chachong_data_sf = chachong_data_sf.sort_values(by=["chachong", aeslist])
        chachong_data_sf = chachong_data_sf[chachong_data_sf.duplicated(subset="chachong", keep=False)]
        # 设置Index
        chachong_data_sf = chachong_data_sf.reset_index(drop=True)
        # 生成重复的数据的例数
        chachong_num = chachong_data_sf["chachong"].value_counts()
        chachong_num = DataFrame(chachong_num)
        chachong_num = chachong_num.rename(columns={0: "chachong"})
        # 生成重复的数据1
        chachong_data_sf_first = chachong_data_sf.drop_duplicates(subset="chachong", keep="first")
        # 生成重复的数据2
        chachong_data_sf_last = chachong_data_sf.drop_duplicates(subset="chachong", keep="last")
        chachong_data_sf_last = chachong_data_sf_last.join(chachong_num, on="chachong", lsuffix="_last", rsuffix="_f")
        # 合并重复列
        # 合并为最后的数据
        zong_sf = chachong_data_sf_last.join(chachong_data_sf_first.set_index("chachong"), on="chachong_last",
                                             lsuffix="_last", rsuffix="_f")
        zong_sf = zong_sf.drop(columns=["name_last", "chachong_last", "name_f"])
        zong_sf = zong_sf.rename(columns={"chachong_f": "重复卡片数"})
        return zong_sf

    ### 读取数据
    # rc_data=read_csv(r"C:/Users/Administrator/Desktop/2015010120181231#reprot.csv",encoding="GB18030")
    rc_data1 = rc_data.copy()
    rc_data1["name"] = hanzi2pinyin(rc_data1["患者姓名"])
    rc_data1["现住地址国标"] = rc_data1["现住地址国标"].map(str)
    rc_data1["报告卡录入时间"] = to_datetime(rc_data1["报告卡录入时间"])

    ## 根据checkbox选择对应的数据列查重
    name_lists = ["name", "有效证件号", "性别", "联系电话", "现住地址国标", "疾病名称", "病例分类", "病例分类2"]
    # check_lists = [CheckVar1.get(), CheckVar2.get(), CheckVar3.get(), CheckVar4.get(), CheckVar5.get(), CheckVar6.get(),CheckVar7.get(), CheckVar8.get()]
    # 身份证选择范围列名字默认为空
    check_lists_sf = [0, 1, CheckVar3.get(), CheckVar4.get(), CheckVar5.get(), CheckVar6.get(), CheckVar7.get(),
                      CheckVar8.get()]
    check_list_pd_sf = DataFrame({"name_lists": name_lists, "check_lists": check_lists_sf})
    check_list_sf = check_list_pd_sf[check_list_pd_sf["check_lists"] == 1]["name_lists"].values.tolist()
    ###排除身份证为空的为数据
    chachong_data_sf = rc_data1.dropna(subset=["有效证件号"])
    zong_sf = shengcheng_chongka(data=chachong_data_sf, checklist=check_list_sf)
    # 通过姓名加其它条件查重
    check_lists_nm = [1, 0, CheckVar3.get(), CheckVar4.get(), CheckVar5.get(), CheckVar6.get(), CheckVar7.get(),
                      CheckVar8.get()]
    check_list_pd_nm = DataFrame({"name_lists": name_lists, "check_lists": check_lists_nm})
    check_list_nm = check_list_pd_nm[check_list_pd_nm["check_lists"] == 1]["name_lists"].values.tolist()
    zong_nm = shengcheng_chongka(data=rc_data1, checklist=check_list_nm)
    ##两个数据的合并
    zong = zong_sf.append(zong_nm)
    zong = zong.drop_duplicates(subset="卡片编号_last", keep="first")
    zong = zong.sort_values(by=["报告单位地区编码_last", "报告卡录入时间_last"], ascending=False)

    try:
        zong_sf.to_csv(entry1.get(), index=False, encoding="GB18030")
        infomessage = "查重完毕！文件保存在{}".format(entry1.get())
        messagebox.showinfo(title="unfortunately ", message=infomessage)
    except:
        # save_error = messagebox.showinfo(title="unfortunately ", message="保存文件出错，请检查！")
        messagebox.showinfo(title="unfortunately ", message="保存文件出错，请检查!")



col_num = 4
button3 = Button(window, text="OK", command=hellook)
canvas.create_window(width * (9 / 18), height_x * col_num, width=80, height=30, window=button3)
col_num = 5
# 基础信息
label3 = Label(window, text="基础信息：", )
canvas.create_window(width * (2 / 18), height_x * col_num, width=170, window=label3)
# 创建选择查重的条件
col_num = 5.8
CheckVar1 = IntVar()
CheckVar2 = IntVar()
CheckVar3 = IntVar()
CheckVar4 = IntVar()
CheckVar5 = IntVar()
C1 = Checkbutton(window, text="姓     名", variable=CheckVar1, onvalue=1, offvalue=0, height=5, width=20,
                 state="disabled")
C1.select()
C2 = Checkbutton(window, text="有效证件号", variable=CheckVar2, onvalue=1, offvalue=0, height=5, width=20, state="disabled")
C2.select()
C3 = Checkbutton(window, text="性     别", variable=CheckVar3, onvalue=1, offvalue=0, height=5, width=20)
C3.select()
C4 = Checkbutton(window, text="联系电话", variable=CheckVar4, onvalue=1, offvalue=0, height=5, width=20)
C4.select()
C5 = Checkbutton(window, text="现住地址国标", variable=CheckVar5, onvalue=1, offvalue=0, height=5, width=20)
C5.select()
# C1.select()
canvas.create_window(width * (2 / 18), height_x * col_num, width=80, height=30, window=C1)
canvas.create_window(width * (5 / 18), height_x * col_num, width=80, height=30, window=C2)
canvas.create_window(width * (8 / 18), height_x * col_num, width=80, height=30, window=C3)
canvas.create_window(width * (11 / 18), height_x * col_num, width=80, height=30, window=C4)
canvas.create_window(width * (14 / 18), height_x * col_num, width=100, height=30, window=C5)

col_num = 6.4
label4 = Label(window, text="疾病信息：", )
canvas.create_window(width * (2 / 18), height_x * col_num, width=170, window=label4)
col_num = 7.2
CheckVar6 = IntVar()
CheckVar7 = IntVar()
CheckVar8 = IntVar()

C6 = Checkbutton(window, text="疾病名称", variable=CheckVar6, onvalue=1, offvalue=0, height=5, width=20, state="disabled")
C6.select()
# C3.select()
C7 = Checkbutton(window, text="病例分类", variable=CheckVar7, onvalue=1, offvalue=0, height=5, width=20)
# C4.select()
C8 = Checkbutton(window, text="病例分类2", variable=CheckVar8, onvalue=1, offvalue=0, height=5, width=20)

canvas.create_window(width * (2 / 18), height_x * col_num, width=80, height=30, window=C6)
canvas.create_window(width * (5 / 18), height_x * col_num, width=80, height=30, window=C7)
canvas.create_window(width * (8 / 18), height_x * col_num, width=80, height=30, window=C8)

col_num = 8
label4 = Label(window, text="问题反馈：ztwenxing@dingtalk.com             源码及说明：https://segmentfault.com/a/1190000018570381", )
canvas.create_window(width * (9 / 18), height_x * col_num,  window=label4)




window.mainloop()