fix(gewu): retry transient sync failures

mjava-gewu/src/main/java/com/malk/gewu/schedule/GWScheduleTask.java

@@ -21,9 +21,10 @@ public class GWScheduleTask {
     private GWService gwService;
 
     /**
-     * 每天凌晨4点同步
+     * 每天凌晨4点05分同步
      */
-    @Scheduled(cron = "0 0 4 * * ? ")
+    // prd: 避开整点时刻的钉钉通讯录接口全局 QPS 高峰
+    @Scheduled(cron = "0 5 4 * * ? ")
     public void syncDingTalkFailedList() {
         try {
             gwService.syncRoster();
@@ -32,4 +33,4 @@ public class GWScheduleTask {
             e.printStackTrace();
         }
     }
-}
+}

mjava-gewu/src/main/java/com/malk/gewu/service/impl/GWImplService.java

@@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSON;
 import com.malk.gewu.service.GWService;
 import com.malk.server.aliwork.YDConf;
 import com.malk.server.aliwork.YDParam;
+import com.malk.server.common.McException;
 import com.malk.server.dingtalk.DDConf;
 import com.malk.service.aliwork.YDClient;
 import com.malk.service.dingtalk.DDClient;
@@ -23,6 +24,15 @@ import java.util.*;
 @Slf4j
 public class GWImplService implements GWService {
 
+    private static final int DEPARTMENT_QUERY_MAX_ATTEMPTS = 3;
+    private static final long DEPARTMENT_QUERY_RETRY_DELAY_MILLIS = 2000L;
+    private static final String DINGTALK_RATE_LIMIT_CODE = "88";
+    private static final String DINGTALK_RATE_LIMIT_SUB_CODE = "subcode=90002";
+    private static final int YIDA_UPDATE_MAX_ATTEMPTS = 3;
+    private static final long YIDA_UPDATE_RETRY_DELAY_MILLIS = 1000L;
+    private static final String YIDA_UNKNOWN_ERROR_CODE = "unknownError";
+    private static final String YIDA_ERROR_SOURCE = "dingtalk_new";
+
     @Autowired
     private DDClient ddClient;
 
@@ -48,7 +58,8 @@ public class GWImplService implements GWService {
         List<Map> metaList = (List<Map>) UtilFile.readJsonObjectFromResource("static/json/personnel"); // 本地匹配了宜搭组件ID
 //        List<Map> metaList = ddClient_personnel.getPersonnelMeta(ddClient.getAccessToken(), ddConf.getAgentId());
         // 同步全量人员
-        ddClient_contacts.getDepartmentId_all(ddClient.getAccessToken(), true).forEach(deptId -> {
+        String accessToken = ddClient.getAccessToken();
+        getDepartmentIdsWithRetry(accessToken).forEach(deptId -> {
             List<String> userIds = ddClient_contacts.listDepartmentUserId(ddClient.getAccessToken(), deptId);
             log.info("dept, {}, userIds, {}", deptId, userIds.size());
             if (userIds.size() == 0) {
@@ -96,7 +107,7 @@ public class GWImplService implements GWService {
                 if (formInstIds.size() > 0) {
                     ydParam.setFormInstanceId(formInstIds.get(0));
                     ydParam.setUpdateFormDataJson(JSON.toJSONString(formData));
-                    ydClient.operateData(ydParam, YDConf.FORM_OPERATION.update);
+                    updateYidaDataWithRetry(ydParam);
                 } else {
                     ydParam.setFormDataJson(JSON.toJSONString(formData));
                     ydClient.operateData(ydParam, YDConf.FORM_OPERATION.create);
@@ -105,6 +116,62 @@ public class GWImplService implements GWService {
         });
     }
 
+    /**
+     * 查询全部部门，针对钉钉通讯录全局 QPS 限流做短暂重试。
+     */
+    private List<Long> getDepartmentIdsWithRetry(String accessToken) {
+        for (int attempt = 1; ; attempt++) {
+            try {
+                return ddClient_contacts.getDepartmentId_all(accessToken, true);
+            } catch (McException exception) {
+                if (!isDepartmentQueryRateLimit(exception) || attempt >= DEPARTMENT_QUERY_MAX_ATTEMPTS) {
+                    throw exception;
+                }
+                log.warn("钉钉部门查询触发限流，等待后重试，attempt={}", attempt);
+                waitBeforeRetry(DEPARTMENT_QUERY_RETRY_DELAY_MILLIS * attempt, "钉钉部门查询");
+            }
+        }
+    }
+
+    private boolean isDepartmentQueryRateLimit(McException exception) {
+        return DINGTALK_RATE_LIMIT_CODE.equals(exception.getCode())
+                && exception.getMessage() != null
+                && exception.getMessage().contains(DINGTALK_RATE_LIMIT_SUB_CODE);
+    }
+
+    /**
+     * 更新实例具备幂等性，仅对宜搭瞬时 unknownError 重试。
+     */
+    private void updateYidaDataWithRetry(YDParam ydParam) {
+        for (int attempt = 1; ; attempt++) {
+            try {
+                ydClient.operateData(ydParam, YDConf.FORM_OPERATION.update);
+                return;
+            } catch (McException exception) {
+                if (!isYidaUnknownError(exception) || attempt >= YIDA_UPDATE_MAX_ATTEMPTS) {
+                    throw exception;
+                }
+                log.warn("宜搭表单实例更新返回瞬时错误，等待后重试，attempt={}", attempt);
+                waitBeforeRetry(YIDA_UPDATE_RETRY_DELAY_MILLIS * attempt, "宜搭表单实例更新");
+            }
+        }
+    }
+
+    private boolean isYidaUnknownError(McException exception) {
+        return YIDA_UNKNOWN_ERROR_CODE.equals(exception.getCode())
+                && YIDA_ERROR_SOURCE.equals(exception.getSource());
+    }
+
+    private void waitBeforeRetry(long delayMillis, String operation) {
+        try {
+            // fixme: 递增等待可避免在外部接口异常窗口内立即重试
+            Thread.sleep(delayMillis);
+        } catch (InterruptedException exception) {
+            Thread.currentThread().interrupt();
+            throw new IllegalStateException("等待重试" + operation + "时线程被中断", exception);
+        }
+    }
+
     @Autowired
     private DDClient_Schedule ddClient_schedule;