|
|
|
@ -10,6 +10,7 @@ from module_admin.entity.do.meta_do import MetadataSuppInfo, MetadataFldSuppInfo |
|
|
|
MetadataFldSuppInfoVett, MetadataExtractInfo, MetadataFldTabExtractInfo |
|
|
|
from module_admin.dao.meta_dao import MetaDao |
|
|
|
from module_admin.dao.datastd_dao import DataStdDao |
|
|
|
from module_admin.dao.user_dao import UserDao |
|
|
|
from datetime import datetime |
|
|
|
from module_admin.entity.vo.user_vo import CurrentUserModel |
|
|
|
from module_admin.entity.vo.approval_vo import ApplyModel |
|
|
|
@ -532,16 +533,20 @@ class MetaService: |
|
|
|
获取元数据导入模板service |
|
|
|
:return: 元数据导入模板excel的二进制数据 |
|
|
|
""" |
|
|
|
table_header_list = ['系统代码', '模式名称', '对象英文名', '补录对象名称', '补录对象描述', '负责人'] |
|
|
|
table_header_list = ['系统代码', '模式名称', '对象英文名', '补录对象名称', '补录对象描述', '对象治理标志', '负责人'] |
|
|
|
column_header_list = ['系统代码', '模式名称', '对象英文名', '字段英文名', '字段补录名', '补录主键', |
|
|
|
'补录字段描述', '引用字典/标准', '安全等级'] |
|
|
|
selector_header_list = ['补录主键', '安全等级'] |
|
|
|
option_list = [{}, {'补录主键': ['是', '否'], '安全等级': ['S1', 'S2', 'S3', 'S4']}] |
|
|
|
option_list = [{'对象治理标志': ['是', '否']}, {'补录主键': ['是', '否'], '安全等级': ['S1', 'S2', 'S3', 'S4']}] |
|
|
|
table_data_list = [['sys_cd', 'mdl_name', 'tab_eng_name', '补录对象中文名', '补录对象描述', '是', 'admin']] |
|
|
|
column_data_list = [['sys_cd', 'mdl_name', 'tab_eng_name', 'fld_eng_name', '字段补录中文名', '是', |
|
|
|
'描述内容', 'dict_code(字典编号)', 'S1']] |
|
|
|
sheet_config1 = dict( |
|
|
|
sheet_name="表信息", header_list=table_header_list, selector_header_list=[] |
|
|
|
sheet_name="表信息", header_list=table_header_list, data_list=table_data_list, selector_header_list=[] |
|
|
|
) |
|
|
|
sheet_config2 = dict( |
|
|
|
sheet_name="字段信息", header_list=column_header_list, selector_header_list=selector_header_list |
|
|
|
sheet_name="字段信息", header_list=column_header_list, data_list=column_data_list, |
|
|
|
selector_header_list=selector_header_list |
|
|
|
) |
|
|
|
sheet_configs = [sheet_config1, sheet_config2] |
|
|
|
binary_data = get_excel_template_with_sheets( |
|
|
|
@ -557,7 +562,6 @@ class MetaService: |
|
|
|
file: UploadFile, |
|
|
|
overWrite: bool, |
|
|
|
current_user: CurrentUserModel): |
|
|
|
import_err_msg = [] |
|
|
|
# 调用DS接口通过系统代码查询系统ID |
|
|
|
url = f'{AppConfig.ds_server_url}/dolphinscheduler/datasources?pageNo=1&pageSize=100' |
|
|
|
headers = {'dashUserName': current_user.user.user_name, 'dashPassword': current_user.user.password} |
|
|
|
@ -570,6 +574,11 @@ class MetaService: |
|
|
|
else: |
|
|
|
raise ServiceException(message=f'系统异常，获取数据源失败') |
|
|
|
skip_table = [] |
|
|
|
result_list = { |
|
|
|
"table": [], |
|
|
|
"column": [], |
|
|
|
"successCount": 0 |
|
|
|
} |
|
|
|
businessId = uuid.uuid4() |
|
|
|
applyTime = datetime.now().strftime("%Y-%m-%d %H:%M:%S") |
|
|
|
table_header_dict = { |
|
|
|
@ -578,6 +587,7 @@ class MetaService: |
|
|
|
'对象英文名': 'tab_eng_name', |
|
|
|
'补录对象名称': 'tab_crrct_name', |
|
|
|
'补录对象描述': 'tab_desc', |
|
|
|
'对象治理标志': 'gov_flag', |
|
|
|
'负责人': 'pic' |
|
|
|
} |
|
|
|
column_header_dict = { |
|
|
|
@ -599,7 +609,6 @@ class MetaService: |
|
|
|
# 逐个读取 |
|
|
|
tableSheet = sheet_names[0] |
|
|
|
columnSheet = sheet_names[1] |
|
|
|
successCount = 0 |
|
|
|
# 校验， 1、各必填内容不能为空， 2.系统代码映射的系统是否存在 3.表是否存在 4.字段是否存在5.引用的字典标准是否存在6.标准的系统与表系统是否对应 |
|
|
|
if tableSheet == '表信息': |
|
|
|
# 表信息补录 |
|
|
|
@ -608,52 +617,85 @@ class MetaService: |
|
|
|
for index, row in df.iterrows(): |
|
|
|
noneValid = '' |
|
|
|
if row['ssys_cd'] is None or len(row['ssys_cd']) == 0: |
|
|
|
noneValid += "sheet[表信息]中行:" + str(index + 2) + "中的系统代码不能为空" |
|
|
|
noneValid += "系统代码不能为空" |
|
|
|
if row['mdl_name'] is None or len(row['mdl_name']) == 0: |
|
|
|
if len(noneValid) > 0: |
|
|
|
noneValid += ",模式名称不能为空" |
|
|
|
else: |
|
|
|
noneValid += "sheet[表信息]中行:" + str(index + 2) + "中的模式名称不能为空" |
|
|
|
noneValid += "模式名称不能为空" |
|
|
|
if row['tab_eng_name'] is None or len(row['tab_eng_name']) == 0: |
|
|
|
if len(noneValid) > 0: |
|
|
|
noneValid += ",表英文名称不能为空" |
|
|
|
noneValid += ",表英文名不能为空" |
|
|
|
else: |
|
|
|
noneValid += "sheet[表信息]中行:" + str(index + 2) + "中的表英文名不能为空" |
|
|
|
noneValid += "表英文名不能为空" |
|
|
|
if len(noneValid) > 0: |
|
|
|
import_err_msg.append(noneValid) |
|
|
|
result_list['table'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"errorInfo": noneValid |
|
|
|
}) |
|
|
|
continue |
|
|
|
ssysId = next((item["id"] for item in dataSourceList if item["name"] == row['ssys_cd']), None) |
|
|
|
if ssysId is None: |
|
|
|
import_err_msg.append("行:" + str(index + 2) + "中的系统不存在，需重新修正") |
|
|
|
result_list['table'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"errorInfo": "系统不存在，为无效系统" |
|
|
|
}) |
|
|
|
continue |
|
|
|
hasTable = await MetaDao.get_lastest_meta_data_supp_vett(result_db, ssysId, row['mdl_name'], |
|
|
|
row['tab_eng_name']) |
|
|
|
if hasTable: |
|
|
|
if hasTable.apply_status == 'waiting': |
|
|
|
import_err_msg.append( |
|
|
|
'sheet[表信息]中行:' + str(index + 2) + ' 导入的系统代码:' + row['ssys_cd'] + ',模式名称:' + |
|
|
|
row['mdl_name'] + |
|
|
|
',对象英文名:' + row['tab_eng_name'] + |
|
|
|
'已存在补录待审核记录，请等待审批完成或撤回申请后，再行导入') |
|
|
|
result_list['table'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"errorInfo": "已存在补录待审核记录，请等待审批完成或撤回申请后，再行导入" |
|
|
|
}) |
|
|
|
skip_table.append({'ssys_id': ssysId, 'mdl_name': row['mdl_name'], |
|
|
|
'tab_eng_name': row['tab_eng_name']}) |
|
|
|
continue |
|
|
|
if hasTable.apply_status == 'pending': |
|
|
|
import_err_msg.append( |
|
|
|
'sheet[字段信息]中行:' + str(index + 2) + ' 导入的系统代码:' + row[ |
|
|
|
'ssys_cd'] + ',模式名称:' + row['mdl_name'] + |
|
|
|
',对象英文名:' + row['tab_eng_name'] + |
|
|
|
'已存在待审核记录，请等待审批完成或撤回申请后，再行导入') |
|
|
|
result_list['table'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"errorInfo": "已存在待审核记录，请等待审批完成或撤回申请后，再行导入" |
|
|
|
}) |
|
|
|
skip_table.append({'ssys_id': ssysId, 'mdl_name': row['mdl_name'], |
|
|
|
'tab_eng_name': row['tab_eng_name']}) |
|
|
|
continue |
|
|
|
|
|
|
|
oldTable = await MetaDao.get_supp_table_by_vett(ssysId, row['mdl_name'], |
|
|
|
row['tab_eng_name'], result_db) |
|
|
|
tableInfo = await MetaDao.get_meta_table(ssysId, row['mdl_name'], |
|
|
|
row['tab_eng_name'], result_db) |
|
|
|
if tableInfo is None: |
|
|
|
import_err_msg.append("sheet[表信息]中行:" + str(index + 2) + "中所对应的表不存在，无法上传补录") |
|
|
|
result_list['table'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"errorInfo": "对应的表不存在，无法上传补录" |
|
|
|
}) |
|
|
|
continue |
|
|
|
if row['pic'] and len(row['pic']) > 0: |
|
|
|
# 校验负责人是否存在 |
|
|
|
user = await UserDao.get_user_by_name(result_db,row['pic']) |
|
|
|
if user is None: |
|
|
|
result_list['table'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"errorInfo": "对应的负责人不存在，无法上传补录" |
|
|
|
}) |
|
|
|
continue |
|
|
|
tableOnum = uuid.uuid4() |
|
|
|
suppTableInfo = MetadataSuppInfoVett() |
|
|
|
@ -664,7 +706,8 @@ class MetaService: |
|
|
|
if overWrite: |
|
|
|
suppTableInfo.tab_crrct_name = row['tab_crrct_name'] |
|
|
|
suppTableInfo.tab_desc = row['tab_desc'] |
|
|
|
suppTableInfo.rec_subm_prsn = row['pic'] |
|
|
|
suppTableInfo.pic = row['pic'] |
|
|
|
suppTableInfo.gov_flag = row['gov_flag'] |
|
|
|
else: |
|
|
|
suppTableInfo.tab_crrct_name = row['tab_crrct_name'] if row['tab_crrct_name'] and \ |
|
|
|
str(row[ |
|
|
|
@ -672,18 +715,20 @@ class MetaService: |
|
|
|
suppTableInfo.tab_desc = row['tab_desc'] if row['tab_desc'] and \ |
|
|
|
str(row[ |
|
|
|
'tab_desc']).strip() != '' else oldTable.tab_desc if oldTable else None |
|
|
|
suppTableInfo.rec_subm_prsn = row['pic'] if row['pic'] and \ |
|
|
|
suppTableInfo.pic = row['pic'] if row['pic'] and \ |
|
|
|
str(row[ |
|
|
|
'pic']).strip() != '' else oldTable.pic if oldTable else None |
|
|
|
suppTableInfo.gov_flag = row['gov_flag'] if row['gov_flag'] and \ |
|
|
|
str(row[ |
|
|
|
'gov_flag']).strip() != '' else oldTable.gov_flag if oldTable else None |
|
|
|
suppTableInfo.rec_subm_prsn = current_user.user.user_name |
|
|
|
suppTableInfo.gov_flag = oldTable.gov_flag if oldTable else None |
|
|
|
suppTableInfo.tab_clas = oldTable.tab_clas if oldTable else None |
|
|
|
suppTableInfo.apply_time = applyTime |
|
|
|
suppTableInfo.apply_status = 'waiting' |
|
|
|
suppTableInfo.business_id = businessId |
|
|
|
suppTableInfo.oldTableData = cls.castToTableStr(oldTable, tableInfo) |
|
|
|
await MetaDao.insertMetadataSuppInfoVett(suppTableInfo, result_db) |
|
|
|
successCount += 1 |
|
|
|
result_list['successCount'] += 1 |
|
|
|
if columnSheet == '字段信息': |
|
|
|
# 字段信息补录 |
|
|
|
df = excel_file.parse(sheet_name=columnSheet, dtype=str, keep_default_na=False, na_values=[]) |
|
|
|
@ -691,19 +736,31 @@ class MetaService: |
|
|
|
for index, row in df.iterrows(): |
|
|
|
noneValid = '' |
|
|
|
if row['ssys_cd'] is None or len(row['ssys_cd']) == 0: |
|
|
|
noneValid += "sheet[字段信息]中行:" + str(index + 2) + "中的系统代码不能为空" |
|
|
|
noneValid += "系统代码不能为空" |
|
|
|
if row['mdl_name'] is None or len(row['mdl_name']) == 0: |
|
|
|
if len(noneValid) > 0: |
|
|
|
noneValid += ",模式名称不能为空" |
|
|
|
else: |
|
|
|
noneValid += "sheet[字段信息]中行:" + str(index + 2) + "中的模式名称不能为空" |
|
|
|
noneValid += "模式名称不能为空" |
|
|
|
if row['tab_eng_name'] is None or len(row['tab_eng_name']) == 0: |
|
|
|
if len(noneValid) > 0: |
|
|
|
noneValid += ",表英文名称不能为空" |
|
|
|
else: |
|
|
|
noneValid += "sheet[字段信息]中行:" + str(index + 2) + "中的表英文名不能为空" |
|
|
|
noneValid += "表英文名不能为空" |
|
|
|
if row['fld_eng_name'] is None or len(row['fld_eng_name']) == 0: |
|
|
|
if len(noneValid) > 0: |
|
|
|
noneValid += ",字段英文名称不能为空" |
|
|
|
else: |
|
|
|
noneValid += "字段英文名不能为空" |
|
|
|
if len(noneValid) > 0: |
|
|
|
import_err_msg.append(noneValid) |
|
|
|
result_list['column'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"fldEngName": row['fld_eng_name'], |
|
|
|
"errorInfo": noneValid |
|
|
|
}) |
|
|
|
continue |
|
|
|
ssysId = next((item["id"] for item in dataSourceList if item["name"] == row['ssys_cd']), None) |
|
|
|
if any( |
|
|
|
@ -724,22 +781,52 @@ class MetaService: |
|
|
|
row['tab_eng_name'], |
|
|
|
row['fld_eng_name'], result_db) |
|
|
|
if columnInfo is None: |
|
|
|
import_err_msg.append("sheet[字段信息]中行:" + str(index + 2) + "中所对应的字段不存在，无法上传补录") |
|
|
|
result_list['column'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"fldEngName": row['fld_eng_name'], |
|
|
|
"errorInfo": "对应的字段不存在，无法上传补录" |
|
|
|
}) |
|
|
|
continue |
|
|
|
dataDictId = '' |
|
|
|
if row['data_dict_id'] and len(row['data_dict_id']) > 0: |
|
|
|
dataDict = await DataStdDao.get_data_dict_by_code(result_db, row['data_dict_id']) |
|
|
|
if dataDict is None: |
|
|
|
import_err_msg.append( |
|
|
|
"sheet[字段信息]中行:" + str(index + 2) + "中所对应的数据字典不存在，无法上传补录") |
|
|
|
result_list['column'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"fldEngName": row['fld_eng_name'], |
|
|
|
"errorInfo": "对应的数据字典不存在，无法上传补录,字段编号:" + row['data_dict_id'] |
|
|
|
}) |
|
|
|
continue |
|
|
|
else: |
|
|
|
if int(dataDict.src_sys) != ssysId: |
|
|
|
import_err_msg.append("sheet[字段信息]中行:" + str( |
|
|
|
index + 2) + "中所对应的数据字典所属系统与表所属系统不一致，无法上传补录") |
|
|
|
result_list['column'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"fldEngName": row['fld_eng_name'], |
|
|
|
"errorInfo": "对应的数据字典所属系统与表所属系统不一致，无法上传补录,字典编号:" + row[ |
|
|
|
'data_dict_id'] |
|
|
|
}) |
|
|
|
continue |
|
|
|
else: |
|
|
|
dataDictId = dataDict.onum |
|
|
|
if row['data_sec_lvl'] and len(row['data_sec_lvl']) and row['data_sec_lvl'] \ |
|
|
|
not in ('S1', 'S2', 'S3', 'S4'): |
|
|
|
result_list['column'].append({ |
|
|
|
"row": index + 2, |
|
|
|
"ssysCd": row['ssys_cd'], |
|
|
|
"mdlName": row['mdl_name'], |
|
|
|
"tabEngName": row['tab_eng_name'], |
|
|
|
"fldEngName": row['fld_eng_name'], |
|
|
|
"errorInfo": "安全等级:" + row['data_sec_lvl'] + "不是S1,S2,S3,S4中的值，无法上传" |
|
|
|
}) |
|
|
|
suppColumnInfo = MetadataFldSuppInfoVett() |
|
|
|
suppColumnInfo.onum = uuid.uuid4() |
|
|
|
suppColumnInfo.ssys_id = ssysId |
|
|
|
@ -776,17 +863,12 @@ class MetaService: |
|
|
|
suppColumnInfo.apply_status = 'waiting' |
|
|
|
suppColumnInfo.oldColumnData = cls.castToColumnStr(oldColumn, columnInfo, data_dict_name) |
|
|
|
await MetaDao.insertMetadataFldSuppInfoVett(suppColumnInfo, result_db) |
|
|
|
successCount += 1 |
|
|
|
if successCount > 0: |
|
|
|
result_list['successCount'] += 1 |
|
|
|
if result_list['successCount'] > 0: |
|
|
|
applyModel = ApplyModel() |
|
|
|
applyModel.businessType = "metaDataInfo" |
|
|
|
applyModel.businessId = businessId |
|
|
|
applyModel.applicant = current_user.user.user_name |
|
|
|
await ApprovalService.apply_services(result_db, applyModel, 'metaDataInfo') |
|
|
|
await result_db.commit() |
|
|
|
else: |
|
|
|
import_err_msg.append("上传的数据均有问题，本次导入0条") |
|
|
|
if len(import_err_msg) > 0: |
|
|
|
return ";".join(import_err_msg) |
|
|
|
else: |
|
|
|
return "操作成功" |
|
|
|
return result_list |
|
|
|
|
xueyinfei
3 weeks ago