diff --git a/api/src/main/java/run/halo/app/core/extension/content/Post.java b/api/src/main/java/run/halo/app/core/extension/content/Post.java index 6a616e9332..6d51ba0edb 100644 --- a/api/src/main/java/run/halo/app/core/extension/content/Post.java +++ b/api/src/main/java/run/halo/app/core/extension/content/Post.java @@ -41,7 +41,8 @@ public class Post extends AbstractExtension { public static final String CATEGORIES_ANNO = "content.halo.run/categories"; public static final String LAST_RELEASED_SNAPSHOT_ANNO = "content.halo.run/last-released-snapshot"; - public static final String TAGS_ANNO = "content.halo.run/tags"; + public static final String LAST_ASSOCIATED_TAGS_ANNO = "content.halo.run/last-associated-tags"; + public static final String DELETED_LABEL = "content.halo.run/deleted"; public static final String PUBLISHED_LABEL = "content.halo.run/published"; public static final String OWNER_LABEL = "content.halo.run/owner"; diff --git a/api/src/main/java/run/halo/app/core/extension/content/Tag.java b/api/src/main/java/run/halo/app/core/extension/content/Tag.java index 7cb047d0d5..ea0654fd63 100644 --- a/api/src/main/java/run/halo/app/core/extension/content/Tag.java +++ b/api/src/main/java/run/halo/app/core/extension/content/Tag.java @@ -27,6 +27,8 @@ public class Tag extends AbstractExtension { public static final GroupVersionKind GVK = GroupVersionKind.fromExtension(Tag.class); + public static final String REQUIRE_SYNC_ON_STARTUP_INDEX_NAME = "requireSyncOnStartup"; + @Schema(requiredMode = REQUIRED) private TagSpec spec; @@ -77,5 +79,7 @@ public static class TagStatus { public Integer visiblePostCount; public Integer postCount; + + private Long observedVersion; } } diff --git a/application/src/main/java/run/halo/app/content/TagPostCountUpdater.java b/application/src/main/java/run/halo/app/content/TagPostCountUpdater.java new file mode 100644 index 0000000000..c6b1d09bdb --- /dev/null +++ b/application/src/main/java/run/halo/app/content/TagPostCountUpdater.java @@ -0,0 +1,186 @@ +package run.halo.app.content; + +import static org.apache.commons.lang3.ObjectUtils.defaultIfNull; +import static run.halo.app.extension.index.query.QueryFactory.and; +import static run.halo.app.extension.index.query.QueryFactory.equal; +import static run.halo.app.extension.index.query.QueryFactory.isNull; + +import com.google.common.collect.Sets; +import java.time.Duration; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.springframework.context.SmartLifecycle; +import org.springframework.context.event.EventListener; +import org.springframework.stereotype.Component; +import run.halo.app.core.extension.content.Post; +import run.halo.app.core.extension.content.Tag; +import run.halo.app.core.extension.content.Tag.TagStatus; +import run.halo.app.event.post.PostDeletedEvent; +import run.halo.app.event.post.PostEvent; +import run.halo.app.event.post.PostUpdatedEvent; +import run.halo.app.extension.ExtensionClient; +import run.halo.app.extension.ListOptions; +import run.halo.app.extension.MetadataUtil; +import run.halo.app.extension.PageRequestImpl; +import run.halo.app.extension.controller.Controller; +import run.halo.app.extension.controller.ControllerBuilder; +import run.halo.app.extension.controller.DefaultController; +import run.halo.app.extension.controller.DefaultQueue; +import run.halo.app.extension.controller.Reconciler; +import run.halo.app.extension.controller.RequestQueue; +import run.halo.app.extension.router.selector.FieldSelector; +import run.halo.app.extension.router.selector.LabelSelector; +import run.halo.app.infra.utils.JsonUtils; + +/** + * Update {@link TagStatus#postCount} when post related to tag is updated. + * + * @author guqing + * @since 2.13.0 + */ +@Component +public class TagPostCountUpdater + implements Reconciler, SmartLifecycle { + + private final RequestQueue tagQueue; + + private final Controller postEventController; + + private final ExtensionClient client; + + private volatile boolean running = false; + + /** + * Construct a {@link TagPostCountUpdater} with the given {@link ExtensionClient}. + */ + public TagPostCountUpdater(ExtensionClient client) { + this.client = client; + + this.tagQueue = new DefaultQueue<>(Instant::now); + this.postEventController = this.setupWith(null); + } + + @Override + public Result reconcile(PostRelatedTags postRelatedTags) { + for (var tag : postRelatedTags.tags()) { + updateTagRelatedPostCount(tag); + } + + // Update last associated tags when handled + client.fetch(Post.class, postRelatedTags.postName()).ifPresent(post -> { + var tags = defaultIfNull(post.getSpec().getTags(), List.of()); + var annotations = MetadataUtil.nullSafeAnnotations(post); + var tagAnno = JsonUtils.objectToJson(tags); + var oldTagAnno = annotations.get(Post.LAST_ASSOCIATED_TAGS_ANNO); + + if (!tagAnno.equals(oldTagAnno)) { + annotations.put(Post.LAST_ASSOCIATED_TAGS_ANNO, tagAnno); + client.update(post); + } + }); + return Result.doNotRetry(); + } + + + @Override + public Controller setupWith(ControllerBuilder builder) { + return new DefaultController<>( + this.getClass().getName(), + this, + tagQueue, + null, + Duration.ofMillis(100), + Duration.ofMinutes(10) + ); + } + + @Override + public void start() { + postEventController.start(); + running = true; + } + + @Override + public void stop() { + running = false; + postEventController.dispose(); + } + + @Override + public boolean isRunning() { + return running; + } + + /** + * Listen to post event to calculate post related to tag for updating. + */ + @EventListener(PostEvent.class) + public void onPostUpdated(PostEvent postEvent) { + var postName = postEvent.getName(); + if (postEvent instanceof PostUpdatedEvent) { + var tagsToUpdate = calcTagsToUpdate(postEvent.getName()); + tagQueue.addImmediately(new PostRelatedTags(postName, tagsToUpdate)); + return; + } + + if (postEvent instanceof PostDeletedEvent deletedEvent) { + var tags = defaultIfNull(deletedEvent.getPost().getSpec().getTags(), + List.of()); + tagQueue.addImmediately(new PostRelatedTags(postName, Sets.newHashSet(tags))); + } + } + + private Set calcTagsToUpdate(String postName) { + var post = client.fetch(Post.class, postName).orElseThrow(); + var annotations = MetadataUtil.nullSafeAnnotations(post); + var oldTags = Optional.ofNullable(annotations.get(Post.LAST_ASSOCIATED_TAGS_ANNO)) + .filter(StringUtils::isNotBlank) + .map(tagsJson -> JsonUtils.jsonToObject(tagsJson, String[].class)) + .orElse(new String[0]); + + var tagsToUpdate = Sets.newHashSet(oldTags); + var newTags = post.getSpec().getTags(); + if (newTags != null) { + tagsToUpdate.addAll(newTags); + } + return tagsToUpdate; + } + + public record PostRelatedTags(String postName, Set tags) { + } + + private void updateTagRelatedPostCount(String tagName) { + client.fetch(Tag.class, tagName).ifPresent(tag -> { + var commonFieldQuery = and( + equal("spec.tags", tag.getMetadata().getName()), + isNull("metadata.deletionTimestamp") + ); + // Update post count + var allPostOptions = new ListOptions(); + allPostOptions.setFieldSelector(FieldSelector.of(commonFieldQuery)); + var result = client.listBy(Post.class, allPostOptions, PageRequestImpl.ofSize(1)); + tag.getStatusOrDefault().setPostCount((int) result.getTotal()); + + // Update visible post count + var publicPostOptions = new ListOptions(); + publicPostOptions.setLabelSelector(LabelSelector.builder() + .eq(Post.PUBLISHED_LABEL, "true") + .build()); + publicPostOptions.setFieldSelector(FieldSelector.of( + and( + commonFieldQuery, + equal("spec.deleted", "false"), + equal("spec.visible", Post.VisibleEnum.PUBLIC.name()) + ) + )); + var publicPosts = + client.listBy(Post.class, publicPostOptions, PageRequestImpl.ofSize(1)); + tag.getStatusOrDefault().setVisiblePostCount((int) publicPosts.getTotal()); + + client.update(tag); + }); + } +} diff --git a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java index aeb56dc574..0a7eec2814 100644 --- a/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java +++ b/application/src/main/java/run/halo/app/core/extension/reconciler/PostReconciler.java @@ -33,6 +33,7 @@ import run.halo.app.core.extension.content.Post.VisibleEnum; import run.halo.app.core.extension.content.Snapshot; import run.halo.app.core.extension.notification.Subscription; +import run.halo.app.event.post.PostDeletedEvent; import run.halo.app.event.post.PostPublishedEvent; import run.halo.app.event.post.PostUnpublishedEvent; import run.halo.app.event.post.PostUpdatedEvent; @@ -86,6 +87,7 @@ public Result reconcile(Request request) { if (ExtensionOperator.isDeleted(post)) { removeFinalizers(post.getMetadata(), Set.of(FINALIZER_NAME)); unPublishPost(post, events); + events.add(new PostDeletedEvent(this, post)); cleanUpResources(post); // update post to be able to be collected by gc collector. client.update(post); @@ -126,7 +128,7 @@ public Result reconcile(Request request) { // calculate the sha256sum var configSha256sum = Hashing.sha256().hashString(post.getSpec().toString(), UTF_8) - .toString(); + .toString(); var oldConfigChecksum = annotations.get(Constant.CHECKSUM_CONFIG_ANNO); if (!Objects.equals(oldConfigChecksum, configSha256sum)) { diff --git a/application/src/main/java/run/halo/app/core/extension/reconciler/TagReconciler.java b/application/src/main/java/run/halo/app/core/extension/reconciler/TagReconciler.java index 3015e6229a..1318e6c28d 100644 --- a/application/src/main/java/run/halo/app/core/extension/reconciler/TagReconciler.java +++ b/application/src/main/java/run/halo/app/core/extension/reconciler/TagReconciler.java @@ -1,30 +1,25 @@ package run.halo.app.core.extension.reconciler; -import static org.apache.commons.lang3.BooleanUtils.isFalse; -import static run.halo.app.extension.MetadataUtil.nullSafeLabels; +import static run.halo.app.extension.ExtensionUtil.addFinalizers; +import static run.halo.app.extension.ExtensionUtil.removeFinalizers; import static run.halo.app.extension.index.query.QueryFactory.equal; -import java.time.Duration; -import java.util.HashSet; import java.util.Map; import java.util.Set; import lombok.RequiredArgsConstructor; import org.apache.commons.lang3.BooleanUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.data.domain.Sort; import org.springframework.stereotype.Component; import run.halo.app.content.permalinks.TagPermalinkPolicy; import run.halo.app.core.extension.content.Constant; -import run.halo.app.core.extension.content.Post; import run.halo.app.core.extension.content.Tag; +import run.halo.app.extension.DefaultExtensionMatcher; import run.halo.app.extension.ExtensionClient; -import run.halo.app.extension.ListOptions; +import run.halo.app.extension.ExtensionUtil; import run.halo.app.extension.MetadataUtil; import run.halo.app.extension.controller.Controller; import run.halo.app.extension.controller.ControllerBuilder; import run.halo.app.extension.controller.Reconciler; import run.halo.app.extension.router.selector.FieldSelector; -import run.halo.app.infra.utils.JsonUtils; /** * Reconciler for {@link Tag}. @@ -35,123 +30,50 @@ @Component @RequiredArgsConstructor public class TagReconciler implements Reconciler { - private static final String FINALIZER_NAME = "tag-protection"; + static final String FINALIZER_NAME = "tag-protection"; private final ExtensionClient client; private final TagPermalinkPolicy tagPermalinkPolicy; @Override public Result reconcile(Request request) { - return client.fetch(Tag.class, request.name()) - .map(tag -> { - if (isDeleted(tag)) { - cleanUpResourcesAndRemoveFinalizer(request.name()); - return Result.doNotRetry(); - } - addFinalizerIfNecessary(tag); - - reconcileMetadata(request.name()); - - this.reconcileStatusPermalink(request.name()); - - reconcileStatusPosts(request.name()); - return new Result(true, Duration.ofMinutes(1)); - }) - .orElse(Result.doNotRetry()); - } - - @Override - public Controller setupWith(ControllerBuilder builder) { - return builder - .syncAllOnStart(true) - .extension(new Tag()) - .build(); - } - - void reconcileMetadata(String name) { - client.fetch(Tag.class, name).ifPresent(tag -> { - Map annotations = MetadataUtil.nullSafeAnnotations(tag); - String oldPermalinkPattern = annotations.get(Constant.PERMALINK_PATTERN_ANNO); - - String newPattern = tagPermalinkPolicy.pattern(); - annotations.put(Constant.PERMALINK_PATTERN_ANNO, newPattern); - - if (!StringUtils.equals(oldPermalinkPattern, newPattern)) { - client.update(tag); - } - }); - } - - private void addFinalizerIfNecessary(Tag oldTag) { - Set finalizers = oldTag.getMetadata().getFinalizers(); - if (finalizers != null && finalizers.contains(FINALIZER_NAME)) { - return; - } - client.fetch(Tag.class, oldTag.getMetadata().getName()) + client.fetch(Tag.class, request.name()) .ifPresent(tag -> { - Set newFinalizers = tag.getMetadata().getFinalizers(); - if (newFinalizers == null) { - newFinalizers = new HashSet<>(); - tag.getMetadata().setFinalizers(newFinalizers); + if (ExtensionUtil.isDeleted(tag)) { + if (removeFinalizers(tag.getMetadata(), Set.of(FINALIZER_NAME))) { + client.update(tag); + } + return; } - newFinalizers.add(FINALIZER_NAME); - client.update(tag); - }); - } - private void cleanUpResourcesAndRemoveFinalizer(String tagName) { - client.fetch(Tag.class, tagName).ifPresent(tag -> { - if (tag.getMetadata().getFinalizers() != null) { - tag.getMetadata().getFinalizers().remove(FINALIZER_NAME); - } - client.update(tag); - }); - } + addFinalizers(tag.getMetadata(), Set.of(FINALIZER_NAME)); - private void reconcileStatusPermalink(String tagName) { - client.fetch(Tag.class, tagName) - .ifPresent(tag -> { - String oldPermalink = tag.getStatusOrDefault().getPermalink(); - String permalink = tagPermalinkPolicy.permalink(tag); - tag.getStatusOrDefault().setPermalink(permalink); + Map annotations = MetadataUtil.nullSafeAnnotations(tag); - if (!StringUtils.equals(permalink, oldPermalink)) { - client.update(tag); - } - }); - } + String newPattern = tagPermalinkPolicy.pattern(); + annotations.put(Constant.PERMALINK_PATTERN_ANNO, newPattern); - private void reconcileStatusPosts(String tagName) { - client.fetch(Tag.class, tagName).ifPresent(tag -> { - Tag oldTag = JsonUtils.deepCopy(tag); + String permalink = tagPermalinkPolicy.permalink(tag); + var status = tag.getStatusOrDefault(); + status.setPermalink(permalink); - populatePosts(tag); + // Update the observed version. + status.setObservedVersion(tag.getMetadata().getVersion() + 1); - if (!oldTag.equals(tag)) { client.update(tag); - } - }); + }); + return Result.doNotRetry(); } - private void populatePosts(Tag tag) { - // populate post-count - var listOptions = new ListOptions(); - listOptions.setFieldSelector(FieldSelector.of( - equal("spec.tags", tag.getMetadata().getName())) - ); - var posts = client.listAll(Post.class, listOptions, Sort.unsorted()); - tag.getStatusOrDefault().setPostCount(posts.size()); - - var publicPosts = posts.stream() - .filter(post -> post.getMetadata().getDeletionTimestamp() == null - && isFalse(post.getSpec().getDeleted()) - && BooleanUtils.TRUE.equals(nullSafeLabels(post).get(Post.PUBLISHED_LABEL)) - && Post.VisibleEnum.PUBLIC.equals(post.getSpec().getVisible()) + @Override + public Controller setupWith(ControllerBuilder builder) { + return builder + .extension(new Tag()) + .onAddMatcher(DefaultExtensionMatcher.builder(client, Tag.GVK) + .fieldSelector(FieldSelector.of( + equal(Tag.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME, BooleanUtils.TRUE)) + ) + .build() ) - .toList(); - tag.getStatusOrDefault().setVisiblePostCount(publicPosts.size()); - } - - private boolean isDeleted(Tag tag) { - return tag.getMetadata().getDeletionTimestamp() != null; + .build(); } } diff --git a/application/src/main/java/run/halo/app/event/post/PostDeletedEvent.java b/application/src/main/java/run/halo/app/event/post/PostDeletedEvent.java new file mode 100644 index 0000000000..37e9c97552 --- /dev/null +++ b/application/src/main/java/run/halo/app/event/post/PostDeletedEvent.java @@ -0,0 +1,21 @@ +package run.halo.app.event.post; + +import lombok.Getter; +import org.springframework.context.ApplicationEvent; +import run.halo.app.core.extension.content.Post; + +@Getter +public class PostDeletedEvent extends ApplicationEvent implements PostEvent { + + private final Post post; + + public PostDeletedEvent(Object source, Post post) { + super(source); + this.post = post; + } + + @Override + public String getName() { + return post.getMetadata().getName(); + } +} diff --git a/application/src/main/java/run/halo/app/infra/SchemeInitializer.java b/application/src/main/java/run/halo/app/infra/SchemeInitializer.java index 649bd3ea80..5b15351a36 100644 --- a/application/src/main/java/run/halo/app/infra/SchemeInitializer.java +++ b/application/src/main/java/run/halo/app/infra/SchemeInitializer.java @@ -197,6 +197,17 @@ public void onApplicationEvent(@NonNull ApplicationContextInitializedEvent event .setName("spec.slug") .setIndexFunc(simpleAttribute(Tag.class, tag -> tag.getSpec().getSlug())) ); + indexSpecs.add(new IndexSpec() + .setName(Tag.REQUIRE_SYNC_ON_STARTUP_INDEX_NAME) + .setIndexFunc(simpleAttribute(Tag.class, tag -> { + var version = tag.getMetadata().getVersion(); + var observedVersion = tag.getStatusOrDefault().getObservedVersion(); + if (observedVersion == null || observedVersion < version) { + return BooleanUtils.TRUE; + } + // do not care about the false case so return null to avoid indexing + return null; + }))); }); schemeManager.register(Snapshot.class, indexSpecs -> { indexSpecs.add(new IndexSpec() diff --git a/application/src/test/java/run/halo/app/core/extension/reconciler/TagReconcilerTest.java b/application/src/test/java/run/halo/app/core/extension/reconciler/TagReconcilerTest.java index 38b1cff16f..d88fe3ca9f 100644 --- a/application/src/test/java/run/halo/app/core/extension/reconciler/TagReconcilerTest.java +++ b/application/src/test/java/run/halo/app/core/extension/reconciler/TagReconcilerTest.java @@ -8,8 +8,8 @@ import static org.mockito.Mockito.when; import java.time.Instant; -import java.util.List; import java.util.Optional; +import java.util.Set; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.mockito.ArgumentCaptor; @@ -17,7 +17,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; import run.halo.app.content.permalinks.TagPermalinkPolicy; -import run.halo.app.core.extension.content.Post; import run.halo.app.core.extension.content.Tag; import run.halo.app.extension.ExtensionClient; import run.halo.app.extension.Metadata; @@ -44,21 +43,20 @@ void reconcile() { Tag tag = tag(); when(client.fetch(eq(Tag.class), eq("fake-tag"))) .thenReturn(Optional.of(tag)); - when(client.listAll(eq(Post.class), any(), any())).thenReturn(List.of()); when(tagPermalinkPolicy.permalink(any())) .thenAnswer(arg -> "/tags/" + tag.getSpec().getSlug()); ArgumentCaptor captor = ArgumentCaptor.forClass(Tag.class); tagReconciler.reconcile(new TagReconciler.Request("fake-tag")); - verify(client, times(3)).update(captor.capture()); + verify(client).update(captor.capture()); Tag capture = captor.getValue(); assertThat(capture.getStatus().getPermalink()).isEqualTo("/tags/fake-slug"); // change slug tag.getSpec().setSlug("new-slug"); tagReconciler.reconcile(new TagReconciler.Request("fake-tag")); - verify(client, times(5)).update(captor.capture()); + verify(client, times(2)).update(captor.capture()); assertThat(capture.getStatus().getPermalink()).isEqualTo("/tags/new-slug"); } @@ -66,6 +64,7 @@ void reconcile() { void reconcileDelete() { Tag tag = tag(); tag.getMetadata().setDeletionTimestamp(Instant.now()); + tag.getMetadata().setFinalizers(Set.of(TagReconciler.FINALIZER_NAME)); when(client.fetch(eq(Tag.class), eq("fake-tag"))) .thenReturn(Optional.of(tag)); ArgumentCaptor captor = ArgumentCaptor.forClass(Tag.class); @@ -75,33 +74,10 @@ void reconcileDelete() { verify(tagPermalinkPolicy, times(0)).permalink(any()); } - @Test - void reconcileStatusPosts() { - Tag tag = tag(); - when(client.fetch(eq(Tag.class), eq("fake-tag"))) - .thenReturn(Optional.of(tag)); - when(client.listAll(eq(Post.class), any(), any())) - .thenReturn(List.of(createPost("fake-post-1"), createPost("fake-post-2"))); - - ArgumentCaptor captor = ArgumentCaptor.forClass(Tag.class); - tagReconciler.reconcile(new TagReconciler.Request("fake-tag")); - verify(client, times(2)).update(captor.capture()); - List allValues = captor.getAllValues(); - assertThat(allValues.get(1).getStatusOrDefault().getPostCount()).isEqualTo(2); - assertThat(allValues.get(1).getStatusOrDefault().getVisiblePostCount()).isEqualTo(0); - } - - Post createPost(String name) { - var post = new Post(); - post.setMetadata(new Metadata()); - post.getMetadata().setName(name); - post.setSpec(new Post.PostSpec()); - return post; - } - Tag tag() { Tag tag = new Tag(); tag.setMetadata(new Metadata()); + tag.getMetadata().setVersion(0L); tag.getMetadata().setName("fake-tag"); tag.setSpec(new Tag.TagSpec());