package com.malk.service.personnel.impl; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONObject; import com.malk.server.aliwork.YDConf; import com.malk.server.aliwork.YDParam; import com.malk.server.dingtalk.DDR_New; import com.malk.server.personnel.PersonnelSyncConf; import com.malk.service.aliwork.YDClient; import com.malk.service.dingtalk.DDClient; import com.malk.service.dingtalk.DDClient_Contacts; import com.malk.service.personnel.PersonnelSyncService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @Slf4j @Service public class PersonnelSyncServiceImpl implements PersonnelSyncService { @Autowired private YDClient ydClient; @Autowired private DDClient ddClient; @Autowired private DDClient_Contacts ddClient_contacts; @Autowired private PersonnelSyncConf conf; private static final String ACTION_CREATE = "CREATE"; private static final String ACTION_UPDATE = "UPDATE"; private static final String ACTION_MARK_OFF = "MARK_OFF"; @Override public Map fullSync(Integer limitOverride) { long start = System.currentTimeMillis(); int limit = effectiveLimit(limitOverride); boolean limited = limit > 0; log.info("[PersonnelSync] 全量同步开始 limit={}", limited ? limit : "none"); List dingUsers = fetchAllDingUsers(); Map dingUserMap = applyLimit(indexByUserid(dingUsers), limit); enrichManagers(dingUserMap); log.info("[PersonnelSync] 钉钉拉取 {} 人 (去重后{})", dingUserMap.size(), limited ? ", 已截断前" + limit + "条" : ""); Map yidaMap = fetchAllYidaPersonnel(); log.info("[PersonnelSync] 宜搭人员档案 {} 条", yidaMap.size()); if (limited) log.info("[PersonnelSync] limit 模式: 本轮跳过 MARK_OFF (拉取非全量)"); List actions = diff(dingUserMap, yidaMap, limited); Map actionStats = countActions(actions); log.info("[PersonnelSync] diff 完成: create={}, update={}, markOff={}, skip={}", actionStats.getOrDefault(ACTION_CREATE, 0L), actionStats.getOrDefault(ACTION_UPDATE, 0L), actionStats.getOrDefault(ACTION_MARK_OFF, 0L), (long) dingUserMap.size() + yidaMap.size() - actions.size()); WriteStats writeStats = concurrentWrite(actions); long cost = System.currentTimeMillis() - start; Map result = new LinkedHashMap<>(); result.put("fetched", dingUserMap.size()); result.put("yidaExisting", yidaMap.size()); result.put("created", writeStats.created.get()); result.put("updated", writeStats.updated.get()); result.put("markedInactive", writeStats.markedInactive.get()); result.put("failed", writeStats.failed.get()); result.put("durationMs", cost); log.info("[PersonnelSync] 全量同步完成 {}", result); return result; } @Override public Map dryRun(Integer limitOverride) { long start = System.currentTimeMillis(); int limit = effectiveLimit(limitOverride); boolean limited = limit > 0; List dingUsers = fetchAllDingUsers(); Map dingUserMap = applyLimit(indexByUserid(dingUsers), limit); enrichManagers(dingUserMap); Map yidaMap = fetchAllYidaPersonnel(); List actions = diff(dingUserMap, yidaMap, limited); Map stats = countActions(actions); Map result = new LinkedHashMap<>(); result.put("fetched", dingUserMap.size()); result.put("yidaExisting", yidaMap.size()); Map actionCounts = new LinkedHashMap<>(); actionCounts.put("create", stats.getOrDefault(ACTION_CREATE, 0L)); actionCounts.put("update", stats.getOrDefault(ACTION_UPDATE, 0L)); actionCounts.put("markOff", stats.getOrDefault(ACTION_MARK_OFF, 0L)); result.put("actions", actionCounts); result.put("durationMs", System.currentTimeMillis() - start); // 抽样 10 条展示预期动作 List> sample = new ArrayList<>(); for (int i = 0; i < Math.min(10, actions.size()); i++) { Action a = actions.get(i); Map s = new LinkedHashMap<>(); s.put("action", a.type); s.put("userid", a.userid); s.put("formData", a.formData); sample.add(s); } result.put("sample", sample); return result; } @Override public Map probeDingtalkUsers(int sampleSize) { long start = System.currentTimeMillis(); String token = ddClient.getAccessToken(); List deptIds = ddClient_contacts.getDepartmentId_all(token, true); List users = ddClient_contacts.getAllUserDetails(token, true); Map byUserid = indexByUserid(users); Map result = new LinkedHashMap<>(); result.put("total", byUserid.size()); result.put("deptCount", deptIds.size()); result.put("durationMs", System.currentTimeMillis() - start); List sample = new ArrayList<>(); int n = Math.min(sampleSize <= 0 ? 3 : sampleSize, users.size()); for (int i = 0; i < n; i++) sample.add(users.get(i)); result.put("sample", sample); return result; } @Override public List fetchAllDingUsers() { String token = ddClient.getAccessToken(); return ddClient_contacts.getAllUserDetails(token, true); } @Override public Map probeSingleUser(String userid) { String token = ddClient.getAccessToken(); return ddClient_contacts.getUserInfoById(token, userid); } @SuppressWarnings("unchecked") public Map probeDiff(String userid) { Map r = new LinkedHashMap<>(); // 钉钉 String token = ddClient.getAccessToken(); Map ding = ddClient_contacts.getUserInfoById(token, userid); r.put("dingUser", ding); // 宜搭 DDR_New result = ydClient.queryData(YDParam.builder() .appType(conf.getYidaAppType()) .systemToken(conf.getYidaSystemToken()) .formUuid(conf.getFormUuidPersonnel()) .searchFieldJson("{\"" + conf.getFieldEmployee() + "\":\"" + userid + "\"}") .build(), YDConf.FORM_QUERY.retrieve_search_form); List list = (List) result.getData(); Map yidaFormData = (list != null && !list.isEmpty()) ? (Map) list.get(0).get("formData") : null; r.put("yidaFormData", yidaFormData); // 新 formData (在职判定改为存在性, probe 统一用 UPDATE) Map newData = toYidaFormData(userid, ding, ACTION_UPDATE); r.put("newFormData", newData); // 字段对比 Map diff = new LinkedHashMap<>(); if (yidaFormData != null) { for (Map.Entry e : newData.entrySet()) { String f = e.getKey(); Object nv = e.getValue(); Object ov = yidaFormData.get(f); Object ovId = yidaFormData.get(f + "_id"); Map d = new LinkedHashMap<>(); d.put("new", nv); d.put("newClass", nv == null ? null : nv.getClass().getSimpleName()); d.put("old", ov); d.put("oldClass", ov == null ? null : ov.getClass().getSimpleName()); d.put("oldId", ovId); d.put("oldIdClass", ovId == null ? null : ovId.getClass().getSimpleName()); diff.put(f, d); } } r.put("diff", diff); return r; } @Override public Map probeStats() { long start = System.currentTimeMillis(); List users = fetchAllDingUsers(); int total = users.size(); int active = 0, inactive = 0, emptyDept = 0, hasExtattr = 0, emptyJobNumber = 0; for (Map u : users) { if (isActive(u)) active++; else inactive++; Object dept = u.get("dept_id_list"); if (!(dept instanceof List) || ((List) dept).isEmpty()) emptyDept++; Object ext = u.get("extattr"); if (ext instanceof Map && !((Map) ext).isEmpty()) hasExtattr++; Object job = u.get("job_number"); if (job == null || String.valueOf(job).isEmpty()) emptyJobNumber++; } Map res = new LinkedHashMap<>(); res.put("total", total); res.put("active", active); res.put("inactive", inactive); res.put("emptyDeptIdList", emptyDept); res.put("hasExtattr", hasExtattr); res.put("emptyJobNumber", emptyJobNumber); res.put("durationMs", System.currentTimeMillis() - start); return res; } // ==================== 内部: 数据抓取 ==================== @SuppressWarnings("unchecked") private Map fetchAllYidaPersonnel() { Map yidaMap = new LinkedHashMap<>(); int currentPage = 1; long totalCount; do { DDR_New result = ydClient.queryData(YDParam.builder() .appType(conf.getYidaAppType()) .systemToken(conf.getYidaSystemToken()) .formUuid(conf.getFormUuidPersonnel()) .currentPage(currentPage) .pageSize(YDConf.PAGE_SIZE_LIMIT) .build(), YDConf.FORM_QUERY.retrieve_search_form); totalCount = result.getTotalCount(); List dataList = (List) result.getData(); if (dataList == null || dataList.isEmpty()) break; for (Map item : dataList) { Map formData = (Map) item.get("formData"); if (formData == null) continue; String userid = extractEmployeeId(formData, conf.getFieldEmployee()); if (userid == null || userid.isEmpty()) continue; YidaRecord rec = new YidaRecord(); rec.instanceId = String.valueOf(item.get("formInstanceId")); rec.formData = formData; yidaMap.put(userid, rec); } currentPage++; } while ((long) (currentPage - 1) * YDConf.PAGE_SIZE_LIMIT < totalCount); return yidaMap; } // ==================== 内部: 差异计算 ==================== private List diff(Map dingUserMap, Map yidaMap, boolean skipMarkOff) { List actions = new ArrayList<>(); // 钉钉里查到的 -> 一律按"在职"写 (active 布尔不再参与) for (Map.Entry e : dingUserMap.entrySet()) { String userid = e.getKey(); Map ding = e.getValue(); YidaRecord yida = yidaMap.get(userid); if (yida == null) { actions.add(new Action(ACTION_CREATE, userid, null, toYidaFormData(userid, ding, ACTION_CREATE))); } else { Map formData = toYidaFormData(userid, ding, ACTION_UPDATE); if (isSameAsYida(formData, yida.formData)) continue; // 幂等跳过 actions.add(new Action(ACTION_UPDATE, userid, yida.instanceId, formData)); } } // 宜搭里有、钉钉里没有 -> 标记离职 (limit 模式下拉取非全量, 跳过此步) if (!skipMarkOff) { for (Map.Entry e : yidaMap.entrySet()) { if (dingUserMap.containsKey(e.getKey())) continue; YidaRecord yida = e.getValue(); Object currentStatus = yida.formData.get(conf.getFieldStatus()); if (conf.getStatusValueInactive().equals(String.valueOf(currentStatus))) continue; // 已离职跳过 Map formData = new LinkedHashMap<>(); formData.put(conf.getFieldStatus(), conf.getStatusValueInactive()); actions.add(new Action(ACTION_MARK_OFF, e.getKey(), yida.instanceId, formData)); } } return actions; } // ==================== 内部: 字段映射 ==================== @SuppressWarnings("unchecked") private Map toYidaFormData(String userid, Map ding, String action) { Map formData = new LinkedHashMap<>(); // 人员 (唯一键, 永远写入) formData.put(conf.getFieldEmployee(), Collections.singletonList(userid)); // 在职状态 String statusValue = ACTION_MARK_OFF.equals(action) ? conf.getStatusValueInactive() : conf.getStatusValueActive(); formData.put(conf.getFieldStatus(), statusValue); // 离职软标记只更新状态字段, 保留其他原值 if (ACTION_MARK_OFF.equals(action)) return formData; // 员工姓名 <- name (目标表该字段 READONLY, 仍按需求强写覆盖) Object name = ding.get("name"); if (notBlank(conf.getFieldName()) && name != null && notBlank(String.valueOf(name))) { formData.put(conf.getFieldName(), String.valueOf(name).trim()); } // 员工编号 <- userid (钉钉用户唯一 ID, 与人员 EmployeeField 同源, 但写入 TextField 便于跨模块按字符串引用) if (notBlank(conf.getFieldJobNumber())) { formData.put(conf.getFieldJobNumber(), userid); } // 员工工号 <- job_number (目标表 READONLY, 按需求强写覆盖; 空则跳过) Object jobNumber = ding.get("job_number"); if (notBlank(conf.getFieldJobNumber2()) && jobNumber != null && notBlank(String.valueOf(jobNumber))) { formData.put(conf.getFieldJobNumber2(), String.valueOf(jobNumber).trim()); } // 员工部门 <- dept_id_list 稳定排序后取首个 (钉钉返回顺序不固定, 避免 diff 抖动) Object deptObj = ding.get("dept_id_list"); if (deptObj instanceof List && !((List) deptObj).isEmpty()) { List sorted = new ArrayList<>(); for (Object d : (List) deptObj) { if (d != null) sorted.add(((Number) d).longValue()); } Collections.sort(sorted); if (!sorted.isEmpty()) { formData.put(conf.getFieldDepartment(), Collections.singletonList(String.valueOf(sorted.get(0)))); } } // 入职时间 <- hired_date (毫秒时间戳, 需要钉钉花名册权限才返回) if (notBlank(conf.getFieldHiredDate())) { Object hired = ding.get("hired_date"); if (hired instanceof Number) { formData.put(conf.getFieldHiredDate(), ((Number) hired).longValue()); } } // Manager <- manager_userid (EmployeeField, 数组格式) Object mgr = ding.get("manager_userid"); if (notBlank(conf.getFieldManager()) && mgr != null && notBlank(String.valueOf(mgr))) { formData.put(conf.getFieldManager(), Collections.singletonList(String.valueOf(mgr).trim())); } // 北森编号 / 归属公司 / 是否CF / 成本中心 <- 钉钉 extattr 自定义字段 putExtAttr(formData, ding, conf.getFieldBeisenJobNo(), conf.getExtAttrKeyBeisen()); putExtAttr(formData, ding, conf.getFieldCompany(), conf.getExtAttrKeyCompany()); putExtAttr(formData, ding, conf.getFieldIsCf(), conf.getExtAttrKeyIsCf()); putExtAttr(formData, ding, conf.getFieldCostCenter(), conf.getExtAttrKeyCostCenter()); // 属性 <- 部门含 externalDeptIds ? 外部 : 内部 (extAttrKeyUserType 留空时走部门白名单) String userType = resolveUserType(ding); if (userType != null) { formData.put(conf.getFieldUserType(), userType); } return formData; } /** extattr 自定义字段取值并写入 (fieldId 或 extKey 为空 / 取不到值 则不写, 保留宜搭原值) */ private void putExtAttr(Map formData, Map ding, String fieldId, String extKey) { if (!notBlank(fieldId) || !notBlank(extKey)) return; String v = readExtAttr(ding, extKey); if (notBlank(v)) formData.put(fieldId, v.trim()); } /** * 读钉钉自定义字段[key]: * - topapi/v2/user/list & user/get 把自定义字段放在 extension (JSON 字符串 {key: value}) 里 * - 兼容旧式 extattr (Map, 值可能是纯字符串或 {text, value} 枚举) */ @SuppressWarnings("unchecked") private String readExtAttr(Map ding, String key) { Object ext = ding.get("extension"); if (ext instanceof String && !((String) ext).trim().isEmpty()) { try { Object parsed = JSON.parse((String) ext); if (parsed instanceof Map) { Object v = ((Map) parsed).get(key); if (v != null && !String.valueOf(v).trim().isEmpty()) return String.valueOf(v); } } catch (Exception ignored) {} } Object extattr = ding.get("extattr"); if (extattr instanceof Map) { Object attr = ((Map) extattr).get(key); if (attr instanceof Map) { Map am = (Map) attr; Object v = am.get("value"); if (v == null) v = am.get("text"); return v == null ? null : String.valueOf(v); } else if (attr != null && !String.valueOf(attr).trim().isEmpty()) { return String.valueOf(attr); } } return null; } private boolean notBlank(String s) { return s != null && !s.trim().isEmpty(); } @SuppressWarnings("unchecked") private String resolveUserType(Map ding) { // 1. 优先读 extattr String key = conf.getExtAttrKeyUserType(); if (key != null && !key.isEmpty()) { Object extattrObj = ding.get("extattr"); if (extattrObj instanceof Map) { Map extattr = (Map) extattrObj; Object attr = extattr.get(key); String raw = null; if (attr instanceof Map) { Map am = (Map) attr; Object value = am.get("value"); if (value == null) value = am.get("text"); if (value != null) raw = String.valueOf(value); } else if (attr != null) { raw = String.valueOf(attr); } if (raw != null && !raw.isEmpty()) { if (conf.getExtAttrValueInternal().equals(raw)) return conf.getExtAttrValueInternal(); if (conf.getExtAttrValueExternal().equals(raw)) return conf.getExtAttrValueExternal(); } } } // 2. 兜底: 部门白名单 (外部部门列表 + 默认规则) List externalDepts = conf.getExternalDeptIds(); Object deptObj = ding.get("dept_id_list"); if (deptObj instanceof List && !((List) deptObj).isEmpty()) { if (externalDepts != null && !externalDepts.isEmpty()) { boolean isExternal = false; for (Object d : (List) deptObj) { if (d == null) continue; long deptId = ((Number) d).longValue(); if (externalDepts.contains(deptId)) { isExternal = true; break; } } return isExternal ? conf.getExtAttrValueExternal() : conf.getExtAttrValueInternal(); } if (conf.isFallbackInternalByDefault()) { return conf.getExtAttrValueInternal(); } } return null; } private boolean isActive(Map ding) { Object active = ding.get("active"); if (active instanceof Boolean) return (Boolean) active; if (active == null) return true; return Boolean.parseBoolean(String.valueOf(active)); } // 比较钉钉侧构造的 formData 与宜搭已有 formData 是否所有字段都相等 @SuppressWarnings("unchecked") private boolean isSameAsYida(Map newData, Map yidaData) { for (Map.Entry e : newData.entrySet()) { String fieldId = e.getKey(); Object newVal = e.getValue(); Object oldVal = yidaData.get(fieldId); if (newVal instanceof List) { // 成员/部门类字段: 宜搭返回时用 _id 后缀取 id 列表 Object oldIdList = yidaData.get(fieldId + "_id"); if (oldIdList != null) oldVal = oldIdList; if (!listEquals((List) newVal, oldVal)) return false; } else { if (!Objects.equals(String.valueOf(newVal), String.valueOf(oldVal))) return false; } } return true; } @SuppressWarnings("unchecked") private boolean listEquals(List newList, Object oldObj) { List newStr = new ArrayList<>(); for (Object o : newList) newStr.add(String.valueOf(o)); List oldStr = new ArrayList<>(); if (oldObj instanceof List) { for (Object o : (List) oldObj) oldStr.add(String.valueOf(o)); } else if (oldObj != null) { oldStr.add(String.valueOf(oldObj)); } if (newStr.size() != oldStr.size()) return false; for (String s : newStr) if (!oldStr.contains(s)) return false; return true; } // ==================== 内部: 写入 ==================== private WriteStats concurrentWrite(List actions) { WriteStats stats = new WriteStats(); if (actions.isEmpty()) return stats; ExecutorService pool = Executors.newFixedThreadPool(Math.max(1, conf.getConcurrency())); List> futures = new ArrayList<>(); for (Action a : actions) { futures.add(pool.submit(() -> executeAction(a, stats))); } for (Future f : futures) { try { f.get(); } catch (Exception ex) { log.warn("[PersonnelSync] future 异常", ex); } } pool.shutdown(); try { pool.awaitTermination(10, TimeUnit.MINUTES); } catch (InterruptedException ignored) {} return stats; } private void executeAction(Action a, WriteStats stats) { int attempt = 0; while (true) { try { if (ACTION_CREATE.equals(a.type)) { ydClient.operateData(YDParam.builder() .appType(conf.getYidaAppType()) .systemToken(conf.getYidaSystemToken()) .formUuid(conf.getFormUuidPersonnel()) .formDataJson(JSON.toJSONString(a.formData)) .build(), YDConf.FORM_OPERATION.create); stats.created.incrementAndGet(); } else { ydClient.operateData(YDParam.builder() .appType(conf.getYidaAppType()) .systemToken(conf.getYidaSystemToken()) .formUuid(conf.getFormUuidPersonnel()) .formInstanceId(a.instanceId) .updateFormDataJson(JSON.toJSONString(a.formData)) .ignoreEmpty(false) .useLatestVersion(true) .build(), YDConf.FORM_OPERATION.update); if (ACTION_MARK_OFF.equals(a.type)) { stats.markedInactive.incrementAndGet(); } else { stats.updated.incrementAndGet(); } } return; } catch (Exception ex) { attempt++; if (attempt > conf.getMaxRetry()) { log.warn("[PersonnelSync] 写入失败 userid={} action={} err={}", a.userid, a.type, ex.getMessage()); stats.failed.incrementAndGet(); return; } } } } // ==================== 内部: 工具 ==================== @SuppressWarnings("unchecked") private String extractEmployeeId(Map formData, String fieldId) { Object raw = formData.get(fieldId + "_id"); if (raw == null) raw = formData.get(fieldId); if (raw == null) return null; if (raw instanceof List) { List list = (List) raw; return list.isEmpty() ? null : String.valueOf(list.get(0)); } String str = String.valueOf(raw).trim(); if (str.startsWith("[") && str.endsWith("]")) { Object parsed = JSON.parse(str); if (parsed instanceof List) { List pl = (List) parsed; return pl.isEmpty() ? null : String.valueOf(pl.get(0)); } } return str.isEmpty() ? null : str; } private Map indexByUserid(List users) { Map map = new LinkedHashMap<>(); for (Map u : users) { Object uid = u.get("userid"); if (uid != null) map.putIfAbsent(String.valueOf(uid), u); } return map; } /** 解析生效的 limit: 入参优先, 否则用配置 limitFirstN; <=0 表示不限 */ private int effectiveLimit(Integer override) { if (override != null) return Math.max(0, override); return Math.max(0, conf.getLimitFirstN()); } /** limit>0 时按 userid 升序取前 N 条 (排序保证幂等可复现) */ private Map applyLimit(Map byUserid, int limit) { if (limit <= 0 || byUserid.size() <= limit) return byUserid; List keys = new ArrayList<>(byUserid.keySet()); Collections.sort(keys); Map limited = new LinkedHashMap<>(); for (int i = 0; i < limit; i++) limited.put(keys.get(i), byUserid.get(keys.get(i))); return limited; } /** topapi/v2/user/list 不返回 manager_userid, 按需逐人补 (仅当 fieldManager 已配置; 单人失败不影响整体) */ @SuppressWarnings("unchecked") private void enrichManagers(Map dingUserMap) { if (!notBlank(conf.getFieldManager()) || dingUserMap.isEmpty()) return; String token = ddClient.getAccessToken(); int filled = 0; for (Map u : dingUserMap.values()) { if (u.get("manager_userid") != null) { filled++; continue; } Object uid = u.get("userid"); if (uid == null) continue; try { Map detail = ddClient_contacts.getUserInfoById(token, String.valueOf(uid)); Object mgr = detail == null ? null : detail.get("manager_userid"); if (mgr != null && notBlank(String.valueOf(mgr))) { u.put("manager_userid", String.valueOf(mgr).trim()); filled++; } } catch (Exception ex) { log.warn("[PersonnelSync] 取 manager_userid 失败 userid={} err={}", uid, ex.getMessage()); } } log.info("[PersonnelSync] manager_userid 已就绪 {}/{} 人", filled, dingUserMap.size()); } private Map countActions(List actions) { Map stats = new HashMap<>(); for (Action a : actions) { stats.merge(a.type, 1L, Long::sum); } return stats; } // ==================== 内部: 数据结构 ==================== private static class YidaRecord { String instanceId; Map formData; } private static class Action { final String type; final String userid; final String instanceId; final Map formData; Action(String type, String userid, String instanceId, Map formData) { this.type = type; this.userid = userid; this.instanceId = instanceId; this.formData = formData; } } private static class WriteStats { AtomicInteger created = new AtomicInteger(); AtomicInteger updated = new AtomicInteger(); AtomicInteger markedInactive = new AtomicInteger(); AtomicInteger failed = new AtomicInteger(); } }