|
@@ -76,15 +76,43 @@ public class ApprovalWriteBackTimer {
|
|
|
AtomicInteger totalScanned = new AtomicInteger(0);
|
|
AtomicInteger totalScanned = new AtomicInteger(0);
|
|
|
AtomicInteger totalTriggered = new AtomicInteger(0);
|
|
AtomicInteger totalTriggered = new AtomicInteger(0);
|
|
|
AtomicInteger totalFailed = new AtomicInteger(0);
|
|
AtomicInteger totalFailed = new AtomicInteger(0);
|
|
|
|
|
+ Map<String, FailedApproval> retryQueue = new LinkedHashMap<>();
|
|
|
|
|
+ Map<String, FailedScan> scanRetryQueue = new LinkedHashMap<>();
|
|
|
|
|
+ Map<String, MissingPage> missingPages = new LinkedHashMap<>();
|
|
|
for (String tag : new String[]{"bill", "other"}) {
|
|
for (String tag : new String[]{"bill", "other"}) {
|
|
|
String formUuid = "bill".equals(tag) ? whConf.getFormUuidApproval() : whConf.getFormUuidOtherApproval();
|
|
String formUuid = "bill".equals(tag) ? whConf.getFormUuidApproval() : whConf.getFormUuidOtherApproval();
|
|
|
if (StringUtils.isBlank(formUuid)) {
|
|
if (StringUtils.isBlank(formUuid)) {
|
|
|
continue;
|
|
continue;
|
|
|
}
|
|
}
|
|
|
- scanForm(tag, formUuid, totalScanned, totalTriggered, totalFailed);
|
|
|
|
|
|
|
+ try {
|
|
|
|
|
+ scanForm(tag, formUuid, 1, totalScanned, totalTriggered, totalFailed, retryQueue, missingPages);
|
|
|
|
|
+ } catch (ScanFailure ex) {
|
|
|
|
|
+ totalFailed.incrementAndGet();
|
|
|
|
|
+ scanRetryQueue.put(tag, new FailedScan(tag, formUuid, ex.page));
|
|
|
|
|
+ log.error("[审批回写兜底] 扫描失败,待本轮结束补扫 tag={} page={}", tag, ex.page, ex);
|
|
|
|
|
+ recordTimerException(ex);
|
|
|
|
|
+ } catch (Exception ex) {
|
|
|
|
|
+ totalFailed.incrementAndGet();
|
|
|
|
|
+ log.error("[审批回写兜底] 扫描失败 tag={}", tag, ex);
|
|
|
|
|
+ recordTimerException(ex);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ for (FailedScan failedScan : scanRetryQueue.values()) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ scanForm(failedScan.tag, failedScan.formUuid, failedScan.page,
|
|
|
|
|
+ totalScanned, totalTriggered, totalFailed, retryQueue, missingPages);
|
|
|
|
|
+ totalFailed.decrementAndGet();
|
|
|
|
|
+ } catch (Exception ex) {
|
|
|
|
|
+ log.error("[审批回写兜底] 本轮补扫仍失败 tag={} page={}",
|
|
|
|
|
+ failedScan.tag, failedScan.page, ex);
|
|
|
|
|
+ recordTimerException(ex);
|
|
|
|
|
+ }
|
|
|
}
|
|
}
|
|
|
- log.info("[审批回写兜底] 完成 scanned={} triggered={} failed={}",
|
|
|
|
|
- totalScanned.get(), totalTriggered.get(), totalFailed.get());
|
|
|
|
|
|
|
+ retryMissingPages(missingPages, totalTriggered, totalFailed, retryQueue);
|
|
|
|
|
+ int recovered = retryFailedApprovals(retryQueue, totalFailed);
|
|
|
|
|
+ log.info("[审批回写兜底] 完成 scanned={} triggered={} scanRetried={} missingPageRetried={} retried={} recovered={} failed={}",
|
|
|
|
|
+ totalScanned.get(), totalTriggered.get(), scanRetryQueue.size(), missingPages.size(),
|
|
|
|
|
+ retryQueue.size(), recovered, totalFailed.get());
|
|
|
} catch (Exception e) {
|
|
} catch (Exception e) {
|
|
|
log.error("[审批回写兜底] 定时任务异常", e);
|
|
log.error("[审批回写兜底] 定时任务异常", e);
|
|
|
recordTimerException(e);
|
|
recordTimerException(e);
|
|
@@ -93,21 +121,29 @@ public class ApprovalWriteBackTimer {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- private void scanForm(String tag, String formUuid,
|
|
|
|
|
- AtomicInteger scanned, AtomicInteger triggered, AtomicInteger failed) {
|
|
|
|
|
|
|
+ private void scanForm(String tag, String formUuid, int startPage,
|
|
|
|
|
+ AtomicInteger scanned, AtomicInteger triggered, AtomicInteger failed,
|
|
|
|
|
+ Map<String, FailedApproval> retryQueue,
|
|
|
|
|
+ Map<String, MissingPage> missingPages) {
|
|
|
// prd 不限制 gmtModified 时间窗,避免服务停机超过窗口后永久漏扫;同步状态非空的历史单仅做内存过滤。
|
|
// prd 不限制 gmtModified 时间窗,避免服务停机超过窗口后永久漏扫;同步状态非空的历史单仅做内存过滤。
|
|
|
// 状态字段: 已知宜搭对 SelectField 支持 IsEmpty 特殊查询,受项目实测差异保守起见改扫全 COMPLETED,内存过滤 syncStatus
|
|
// 状态字段: 已知宜搭对 SelectField 支持 IsEmpty 特殊查询,受项目实测差异保守起见改扫全 COMPLETED,内存过滤 syncStatus
|
|
|
- int page = 1;
|
|
|
|
|
|
|
+ int page = startPage;
|
|
|
while (true) {
|
|
while (true) {
|
|
|
- DDR_New r = ydClient.queryData(YDParam.builder()
|
|
|
|
|
- .appType(whConf.getYidaAppType())
|
|
|
|
|
- .systemToken(whConf.getYidaSystemToken())
|
|
|
|
|
- .formUuid(formUuid)
|
|
|
|
|
- .instanceStatus("COMPLETED")
|
|
|
|
|
- .pageNumber(page)
|
|
|
|
|
- .pageSize(PAGE_SIZE)
|
|
|
|
|
- .build(), YDConf.FORM_QUERY.retrieve_search_process);
|
|
|
|
|
- List<Map> data = (List<Map>) r.getData();
|
|
|
|
|
|
|
+ DDR_New response;
|
|
|
|
|
+ List<Map> data;
|
|
|
|
|
+ try {
|
|
|
|
|
+ response = ydClient.queryData(YDParam.builder()
|
|
|
|
|
+ .appType(whConf.getYidaAppType())
|
|
|
|
|
+ .systemToken(whConf.getYidaSystemToken())
|
|
|
|
|
+ .formUuid(formUuid)
|
|
|
|
|
+ .instanceStatus("COMPLETED")
|
|
|
|
|
+ .pageNumber(page)
|
|
|
|
|
+ .pageSize(PAGE_SIZE)
|
|
|
|
|
+ .build(), YDConf.FORM_QUERY.retrieve_search_process);
|
|
|
|
|
+ data = (List<Map>) response.getData();
|
|
|
|
|
+ } catch (Exception ex) {
|
|
|
|
|
+ throw new ScanFailure(page, ex);
|
|
|
|
|
+ }
|
|
|
if (data == null || data.isEmpty()) {
|
|
if (data == null || data.isEmpty()) {
|
|
|
break;
|
|
break;
|
|
|
}
|
|
}
|
|
@@ -121,6 +157,7 @@ public class ApprovalWriteBackTimer {
|
|
|
Map fd = extractFormData(item);
|
|
Map fd = extractFormData(item);
|
|
|
if (fd == null) {
|
|
if (fd == null) {
|
|
|
failed.incrementAndGet();
|
|
failed.incrementAndGet();
|
|
|
|
|
+ queueIncompletePage(missingPages, tag, formUuid, page, finstId);
|
|
|
log.warn("[审批回写兜底] 流程实例缺少表单数据 keys={}", item.keySet());
|
|
log.warn("[审批回写兜底] 流程实例缺少表单数据 keys={}", item.keySet());
|
|
|
recordMissingFormData(tag, finstId, item);
|
|
recordMissingFormData(tag, finstId, item);
|
|
|
continue;
|
|
continue;
|
|
@@ -133,7 +170,8 @@ public class ApprovalWriteBackTimer {
|
|
|
Integer resultCode = resolveResultCode(item.get("approvedResult"));
|
|
Integer resultCode = resolveResultCode(item.get("approvedResult"));
|
|
|
if (resultCode == null) {
|
|
if (resultCode == null) {
|
|
|
failed.incrementAndGet();
|
|
failed.incrementAndGet();
|
|
|
- log.warn("[审批回写兜底] 审批结果缺失或未知,跳过并等待下轮 tag={} formInstanceId={} approvedResult={}",
|
|
|
|
|
|
|
+ queueIncompletePage(missingPages, tag, formUuid, page, finstId);
|
|
|
|
|
+ log.warn("[审批回写兜底] 审批结果缺失或未知,待本轮结束补扫 tag={} formInstanceId={} approvedResult={}",
|
|
|
tag, finstId, item.get("approvedResult"));
|
|
tag, finstId, item.get("approvedResult"));
|
|
|
continue;
|
|
continue;
|
|
|
}
|
|
}
|
|
@@ -143,23 +181,204 @@ public class ApprovalWriteBackTimer {
|
|
|
triggered.incrementAndGet();
|
|
triggered.incrementAndGet();
|
|
|
if (result == null || !"全部成功".equals(result.getSyncStatus())) {
|
|
if (result == null || !"全部成功".equals(result.getSyncStatus())) {
|
|
|
failed.incrementAndGet();
|
|
failed.incrementAndGet();
|
|
|
|
|
+ retryQueue.put(finstId, new FailedApproval(tag, finstId, resultCode));
|
|
|
recordWriteBackFailure(tag, finstId, result);
|
|
recordWriteBackFailure(tag, finstId, result);
|
|
|
}
|
|
}
|
|
|
log.info("[审批回写兜底] 触发 writeBack tag={} formInstanceId={} result={}",
|
|
log.info("[审批回写兜底] 触发 writeBack tag={} formInstanceId={} result={}",
|
|
|
tag, finstId, resultCode);
|
|
tag, finstId, resultCode);
|
|
|
} catch (Exception ex) {
|
|
} catch (Exception ex) {
|
|
|
failed.incrementAndGet();
|
|
failed.incrementAndGet();
|
|
|
|
|
+ retryQueue.put(finstId, new FailedApproval(tag, finstId, resultCode));
|
|
|
log.error("[审批回写兜底] writeBack 失败 tag={} formInstanceId={}", tag, finstId, ex);
|
|
log.error("[审批回写兜底] writeBack 失败 tag={} formInstanceId={}", tag, finstId, ex);
|
|
|
recordWriteBackException(tag, finstId, ex);
|
|
recordWriteBackException(tag, finstId, ex);
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
- if (!hasNextPage(page, PAGE_SIZE, r.getTotalCount(), data.size())) {
|
|
|
|
|
|
|
+ if (!hasNextPage(page, PAGE_SIZE, response.getTotalCount(), data.size())) {
|
|
|
break;
|
|
break;
|
|
|
}
|
|
}
|
|
|
page++;
|
|
page++;
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ private static final class FailedScan {
|
|
|
|
|
+ private final String tag;
|
|
|
|
|
+ private final String formUuid;
|
|
|
|
|
+ private final int page;
|
|
|
|
|
+
|
|
|
|
|
+ private FailedScan(String tag, String formUuid, int page) {
|
|
|
|
|
+ this.tag = tag;
|
|
|
|
|
+ this.formUuid = formUuid;
|
|
|
|
|
+ this.page = page;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static final class ScanFailure extends RuntimeException {
|
|
|
|
|
+ private final int page;
|
|
|
|
|
+
|
|
|
|
|
+ private ScanFailure(int page, Exception cause) {
|
|
|
|
|
+ super(cause);
|
|
|
|
|
+ this.page = page;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static final class MissingPage {
|
|
|
|
|
+ private final String tag;
|
|
|
|
|
+ private final String formUuid;
|
|
|
|
|
+ private final int page;
|
|
|
|
|
+ private final Set<String> formInstanceIds = new LinkedHashSet<>();
|
|
|
|
|
+
|
|
|
|
|
+ private MissingPage(String tag, String formUuid, int page) {
|
|
|
|
|
+ this.tag = tag;
|
|
|
|
|
+ this.formUuid = formUuid;
|
|
|
|
|
+ this.page = page;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static void queueIncompletePage(Map<String, MissingPage> missingPages,
|
|
|
|
|
+ String tag, String formUuid, int page,
|
|
|
|
|
+ String formInstanceId) {
|
|
|
|
|
+ MissingPage missingPage = missingPages.get(tag + ":" + page);
|
|
|
|
|
+ if (missingPage == null) {
|
|
|
|
|
+ missingPage = new MissingPage(tag, formUuid, page);
|
|
|
|
|
+ missingPages.put(tag + ":" + page, missingPage);
|
|
|
|
|
+ }
|
|
|
|
|
+ missingPage.formInstanceIds.add(formInstanceId);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ @SuppressWarnings("unchecked")
|
|
|
|
|
+ private void retryMissingPages(Map<String, MissingPage> missingPages,
|
|
|
|
|
+ AtomicInteger triggered, AtomicInteger failed,
|
|
|
|
|
+ Map<String, FailedApproval> retryQueue) {
|
|
|
|
|
+ for (MissingPage missingPage : missingPages.values()) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ DDR_New response = ydClient.queryData(YDParam.builder()
|
|
|
|
|
+ .appType(whConf.getYidaAppType())
|
|
|
|
|
+ .systemToken(whConf.getYidaSystemToken())
|
|
|
|
|
+ .formUuid(missingPage.formUuid)
|
|
|
|
|
+ .instanceStatus("COMPLETED")
|
|
|
|
|
+ .pageNumber(missingPage.page)
|
|
|
|
|
+ .pageSize(PAGE_SIZE)
|
|
|
|
|
+ .build(), YDConf.FORM_QUERY.retrieve_search_process);
|
|
|
|
|
+ List<Map> items = (List<Map>) response.getData();
|
|
|
|
|
+ Set<String> unresolved = new LinkedHashSet<>(missingPage.formInstanceIds);
|
|
|
|
|
+ if (items != null) {
|
|
|
|
|
+ for (Map item : items) {
|
|
|
|
|
+ String formInstanceId = formInstanceId(item);
|
|
|
|
|
+ if (!unresolved.contains(formInstanceId)) continue;
|
|
|
|
|
+ if (retryIncompleteItem(missingPage.tag, item, triggered, failed, retryQueue)) {
|
|
|
|
|
+ unresolved.remove(formInstanceId);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ for (String formInstanceId : unresolved) {
|
|
|
|
|
+ log.error("[审批回写兜底] 本轮补扫仍缺少表单数据、审批结果或目标实例 tag={} formInstanceId={}",
|
|
|
|
|
+ missingPage.tag, formInstanceId);
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (Exception ex) {
|
|
|
|
|
+ log.error("[审批回写兜底] 缺数据页面本轮补扫失败 tag={} page={}",
|
|
|
|
|
+ missingPage.tag, missingPage.page, ex);
|
|
|
|
|
+ recordTimerException(ex);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private boolean retryIncompleteItem(String tag, Map item,
|
|
|
|
|
+ AtomicInteger triggered, AtomicInteger failed,
|
|
|
|
|
+ Map<String, FailedApproval> retryQueue) {
|
|
|
|
|
+ Map formData = extractFormData(item);
|
|
|
|
|
+ if (formData == null) return false;
|
|
|
|
|
+ String formInstanceId = formInstanceId(item);
|
|
|
|
|
+ String status = String.valueOf(formData.getOrDefault(resolveSyncStatusField(tag), ""));
|
|
|
|
|
+ if (!shouldRetrySyncStatus(status)) {
|
|
|
|
|
+ failed.decrementAndGet();
|
|
|
|
|
+ return true;
|
|
|
|
|
+ }
|
|
|
|
|
+ Integer resultCode = resolveResultCode(item.get("approvedResult"));
|
|
|
|
|
+ if (resultCode == null) return false;
|
|
|
|
|
+ try {
|
|
|
|
|
+ ApprovalWriteBackResult result = approvalWriteBackService.writeBack(formInstanceId, resultCode);
|
|
|
|
|
+ triggered.incrementAndGet();
|
|
|
|
|
+ if (result != null && "全部成功".equals(result.getSyncStatus())) {
|
|
|
|
|
+ failed.decrementAndGet();
|
|
|
|
|
+ } else {
|
|
|
|
|
+ retryQueue.put(formInstanceId, new FailedApproval(tag, formInstanceId, resultCode));
|
|
|
|
|
+ recordWriteBackFailure(tag, formInstanceId, result);
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (Exception ex) {
|
|
|
|
|
+ retryQueue.put(formInstanceId, new FailedApproval(tag, formInstanceId, resultCode));
|
|
|
|
|
+ recordWriteBackException(tag, formInstanceId, ex);
|
|
|
|
|
+ }
|
|
|
|
|
+ return true;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static String formInstanceId(Map item) {
|
|
|
|
|
+ Object instanceId = item.get("formInstanceId");
|
|
|
|
|
+ if (instanceId == null) instanceId = item.get("processInstanceId");
|
|
|
|
|
+ return instanceId == null ? "" : String.valueOf(instanceId);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private int retryFailedApprovals(Map<String, FailedApproval> retryQueue, AtomicInteger failed) {
|
|
|
|
|
+ int recovered = 0;
|
|
|
|
|
+ for (FailedApproval target : retryQueue.values()) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ if (approvalStatusAlreadySuccess(target)) {
|
|
|
|
|
+ failed.decrementAndGet();
|
|
|
|
|
+ recovered++;
|
|
|
|
|
+ continue;
|
|
|
|
|
+ }
|
|
|
|
|
+ ApprovalWriteBackResult result = approvalWriteBackService.writeBack(
|
|
|
|
|
+ target.formInstanceId, target.resultCode);
|
|
|
|
|
+ if (result != null && "全部成功".equals(result.getSyncStatus())) {
|
|
|
|
|
+ failed.decrementAndGet();
|
|
|
|
|
+ recovered++;
|
|
|
|
|
+ log.info("[审批回写兜底] 延后重试成功 tag={} formInstanceId={}",
|
|
|
|
|
+ target.tag, target.formInstanceId);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ log.warn("[审批回写兜底] 延后重试仍未完成 tag={} formInstanceId={}",
|
|
|
|
|
+ target.tag, target.formInstanceId);
|
|
|
|
|
+ recordWriteBackFailure(target.tag, target.formInstanceId, result);
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (Exception ex) {
|
|
|
|
|
+ log.error("[审批回写兜底] 延后重试失败 tag={} formInstanceId={}",
|
|
|
|
|
+ target.tag, target.formInstanceId, ex);
|
|
|
|
|
+ recordWriteBackException(target.tag, target.formInstanceId, ex);
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ return recovered;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private boolean approvalStatusAlreadySuccess(FailedApproval target) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ DDR_New response = ydClient.queryData(YDParam.builder()
|
|
|
|
|
+ .appType(whConf.getYidaAppType())
|
|
|
|
|
+ .systemToken(whConf.getYidaSystemToken())
|
|
|
|
|
+ .formInstanceId(target.formInstanceId)
|
|
|
|
|
+ .build(), YDConf.FORM_QUERY.retrieve_id);
|
|
|
|
|
+ Map formData = response == null ? null : response.getFormData();
|
|
|
|
|
+ if (formData == null && response != null && response.getData() instanceof Map) {
|
|
|
|
|
+ formData = (Map) response.getData();
|
|
|
|
|
+ }
|
|
|
|
|
+ return formData != null && "全部成功".equals(String.valueOf(
|
|
|
|
|
+ formData.get(resolveSyncStatusField(target.tag))));
|
|
|
|
|
+ } catch (Exception ex) {
|
|
|
|
|
+ log.warn("[审批回写兜底] 补试前状态回读失败,继续执行幂等回写 formInstanceId={}",
|
|
|
|
|
+ target.formInstanceId, ex);
|
|
|
|
|
+ return false;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private static final class FailedApproval {
|
|
|
|
|
+ private final String tag;
|
|
|
|
|
+ private final String formInstanceId;
|
|
|
|
|
+ private final int resultCode;
|
|
|
|
|
+
|
|
|
|
|
+ private FailedApproval(String tag, String formInstanceId, int resultCode) {
|
|
|
|
|
+ this.tag = tag;
|
|
|
|
|
+ this.formInstanceId = formInstanceId;
|
|
|
|
|
+ this.resultCode = resultCode;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
// fixme 不设置固定页数上限,避免全量扫描超过 2000 条时漏掉历史未回写审批。
|
|
// fixme 不设置固定页数上限,避免全量扫描超过 2000 条时漏掉历史未回写审批。
|
|
|
static boolean hasNextPage(int pageNumber, int pageSize, long totalCount, int returnedSize) {
|
|
static boolean hasNextPage(int pageNumber, int pageSize, long totalCount, int returnedSize) {
|
|
|
if (returnedSize < pageSize) {
|
|
if (returnedSize < pageSize) {
|