From f964b2c185df5e73d494c068ef9b3c6c0f9407d6 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Sun, 14 Apr 2024 01:56:30 -0500 Subject: [PATCH 01/16] Start to add new stats tables --- migrations/0055_stats.sql | 65 ++++++++++++++++++++++++++ src/controllers/episode.ts | 6 --- src/controllers/podcast.ts | 6 --- src/entities/episode.ts | 54 +++------------------ src/entities/episodes_most_recent.ts | 18 ------- src/entities/index.ts | 3 ++ src/entities/mediaRef.ts | 59 +++++++---------------- src/entities/mediaRef_videos.ts | 18 ------- src/entities/podcast.ts | 70 ++++------------------------ src/entities/statsEpisode.ts | 56 ++++++++++++++++++++++ src/entities/statsMediaRef.ts | 56 ++++++++++++++++++++++ src/entities/statsPodcast.ts | 56 ++++++++++++++++++++++ src/lib/db.ts | 6 +++ src/seeds/qa/stats.ts | 36 +++++++------- src/services/stats.ts | 1 - 15 files changed, 289 insertions(+), 221 deletions(-) create mode 100644 migrations/0055_stats.sql create mode 100644 src/entities/statsEpisode.ts create mode 100644 src/entities/statsMediaRef.ts create mode 100644 src/entities/statsPodcast.ts diff --git a/migrations/0055_stats.sql b/migrations/0055_stats.sql new file mode 100644 index 00000000..f8071cb0 --- /dev/null +++ b/migrations/0055_stats.sql @@ -0,0 +1,65 @@ +-- Podcasts Stats +CREATE TABLE IF NOT EXISTS stats_podcast ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + "pastDayTotalUniquePageviews" INTEGER DEFAULT 0, + "pastWeekTotalUniquePageviews" INTEGER DEFAULT 0, + "pastMonthTotalUniquePageviews" INTEGER DEFAULT 0, + "pastYearTotalUniquePageviews" INTEGER DEFAULT 0, + "pastAllTimeTotalUniquePageviews" INTEGER DEFAULT 0, + podcast_id VARCHAR NOT NULL UNIQUE REFERENCES podcasts(id) ON DELETE CASCADE, + "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX "stats_podcast_pastDayTotalUniquePageviews_idx" ON stats_podcast ("pastDayTotalUniquePageviews"); +CREATE INDEX "stats_podcast_pastWeekTotalUniquePageviews_idx" ON stats_podcast ("pastWeekTotalUniquePageviews"); +CREATE INDEX "stats_podcast_pastMonthTotalUniquePageviews_idx" ON stats_podcast ("pastMonthTotalUniquePageviews"); +CREATE INDEX "stats_podcast_pastYearTotalUniquePageviews_idx" ON stats_podcast ("pastYearTotalUniquePageviews"); +CREATE INDEX "stats_podcast_pastAllTimeTotalUniquePageviews_idx" ON stats_podcast ("pastAllTimeTotalUniquePageviews"); + +ALTER TABLE podcasts +ADD COLUMN stats_podcast_id VARCHAR UNIQUE; + +-- Episodes Stats +CREATE TABLE IF NOT EXISTS stats_episode ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + "pastDayTotalUniquePageviews" INTEGER DEFAULT 0, + "pastWeekTotalUniquePageviews" INTEGER DEFAULT 0, + "pastMonthTotalUniquePageviews" INTEGER DEFAULT 0, + "pastYearTotalUniquePageviews" INTEGER DEFAULT 0, + "pastAllTimeTotalUniquePageviews" INTEGER DEFAULT 0, + episode_id VARCHAR NOT NULL UNIQUE REFERENCES episodes(id) ON DELETE CASCADE, + "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX "stats_episode_pastDayTotalUniquePageviews_idx" ON stats_episode ("pastDayTotalUniquePageviews"); +CREATE INDEX "stats_episode_pastWeekTotalUniquePageviews_idx" ON stats_episode ("pastWeekTotalUniquePageviews"); +CREATE INDEX "stats_episode_pastMonthTotalUniquePageviews_idx" ON stats_episode ("pastMonthTotalUniquePageviews"); +CREATE INDEX "stats_episode_pastYearTotalUniquePageviews_idx" ON stats_episode ("pastYearTotalUniquePageviews"); +CREATE INDEX "stats_episode_pastAllTimeTotalUniquePageviews_idx" ON stats_episode ("pastAllTimeTotalUniquePageviews"); + +ALTER TABLE episodes +ADD COLUMN stats_episode_id VARCHAR UNIQUE; + +-- Media Refs Stats +CREATE TABLE IF NOT EXISTS stats_media_ref ( + id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), + "pastDayTotalUniquePageviews" INTEGER DEFAULT 0, + "pastWeekTotalUniquePageviews" INTEGER DEFAULT 0, + "pastMonthTotalUniquePageviews" INTEGER DEFAULT 0, + "pastYearTotalUniquePageviews" INTEGER DEFAULT 0, + "pastAllTimeTotalUniquePageviews" INTEGER DEFAULT 0, + media_ref_id VARCHAR NOT NULL UNIQUE REFERENCES "mediaRefs"(id) ON DELETE CASCADE, + "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, + "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX "stats_media_ref_pastDayTotalUniquePageviews_idx" ON stats_media_ref ("pastDayTotalUniquePageviews"); +CREATE INDEX "stats_media_ref_pastWeekTotalUniquePageviews_idx" ON stats_media_ref ("pastWeekTotalUniquePageviews"); +CREATE INDEX "stats_media_ref_pastMonthTotalUniquePageviews_idx" ON stats_media_ref ("pastMonthTotalUniquePageviews"); +CREATE INDEX "stats_media_ref_pastYearTotalUniquePageviews_idx" ON stats_media_ref ("pastYearTotalUniquePageviews"); +CREATE INDEX "stats_media_ref_pastAllTimeTotalUniquePageviews_idx" ON stats_media_ref ("pastAllTimeTotalUniquePageviews"); + +ALTER TABLE "mediaRefs" +ADD COLUMN stats_media_ref_id VARCHAR UNIQUE; diff --git a/src/controllers/episode.ts b/src/controllers/episode.ts index d6fbf6f2..eb718adc 100644 --- a/src/controllers/episode.ts +++ b/src/controllers/episode.ts @@ -133,12 +133,6 @@ const addSelectsToQueryBuilder = (qb) => { .addSelect('episode.mediaFilesize') .addSelect('episode.mediaType') .addSelect('episode.mediaUrl') - .addSelect('episode.pastHourTotalUniquePageviews') - .addSelect('episode.pastDayTotalUniquePageviews') - .addSelect('episode.pastWeekTotalUniquePageviews') - .addSelect('episode.pastMonthTotalUniquePageviews') - .addSelect('episode.pastYearTotalUniquePageviews') - .addSelect('episode.pastAllTimeTotalUniquePageviews') .addSelect('episode.pubDate') .addSelect('episode.socialInteraction') .addSelect('episode.subtitle') diff --git a/src/controllers/podcast.ts b/src/controllers/podcast.ts index 20d378e9..cc077c0a 100644 --- a/src/controllers/podcast.ts +++ b/src/controllers/podcast.ts @@ -226,12 +226,6 @@ const getPodcasts = async (query, countOverride?, isFromManticoreSearch?) => { .addSelect('podcast.latestLiveItemStatus') .addSelect('podcast.linkUrl') .addSelect('podcast.medium') - .addSelect('podcast.pastHourTotalUniquePageviews') - .addSelect('podcast.pastWeekTotalUniquePageviews') - .addSelect('podcast.pastDayTotalUniquePageviews') - .addSelect('podcast.pastMonthTotalUniquePageviews') - .addSelect('podcast.pastYearTotalUniquePageviews') - .addSelect('podcast.pastAllTimeTotalUniquePageviews') .addSelect('podcast.shrunkImageUrl') .addSelect('podcast.sortableTitle') .addSelect('podcast.subtitle') diff --git a/src/entities/episode.ts b/src/entities/episode.ts index d2ef9346..7e027fa4 100644 --- a/src/entities/episode.ts +++ b/src/entities/episode.ts @@ -15,6 +15,7 @@ import { LiveItem, MediaRef, Podcast, + StatsEpisode, UserHistoryItem, UserNowPlayingItem, UserQueueItem @@ -26,6 +27,7 @@ import { CreateDateColumn, Entity, Index, + JoinColumn, JoinTable, ManyToMany, ManyToOne, @@ -38,12 +40,6 @@ import { generateShortId } from '~/lib/utility' @Entity('episodes') @Index(['isPublic', 'pubDate']) -@Index(['mediaType', 'pastAllTimeTotalUniquePageviews']) -@Index(['mediaType', 'pastHourTotalUniquePageviews']) -@Index(['mediaType', 'pastDayTotalUniquePageviews']) -@Index(['mediaType', 'pastWeekTotalUniquePageviews']) -@Index(['mediaType', 'pastMonthTotalUniquePageviews']) -@Index(['mediaType', 'pastYearTotalUniquePageviews']) export class Episode { @PrimaryColumn('varchar', { default: generateShortId(), @@ -139,48 +135,6 @@ export class Episode { @Column() mediaUrl: string - @Index() - @ValidateIf((a) => a.pastHourTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastHourTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastDayTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastWeekTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastMonthTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastYearTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastAllTimeTotalUniquePageviews: number - @Column({ nullable: true }) pubDate?: Date @@ -237,6 +191,10 @@ export class Episode { @OneToMany((type) => UserQueueItem, (userQueueItem) => userQueueItem.episode) userQueueItems: UserQueueItem[] + @OneToOne(() => StatsEpisode, { nullable: true }) + @JoinColumn({ name: 'stats_episode_id' }) + stats_episode?: StatsEpisode + @CreateDateColumn() createdAt: Date diff --git a/src/entities/episodes_most_recent.ts b/src/entities/episodes_most_recent.ts index f5766845..fb751cf5 100644 --- a/src/entities/episodes_most_recent.ts +++ b/src/entities/episodes_most_recent.ts @@ -81,24 +81,6 @@ export class EpisodeMostRecent { @ViewColumn() mediaUrl: string - @ViewColumn() - pastHourTotalUniquePageviews: number - - @ViewColumn() - pastDayTotalUniquePageviews: number - - @ViewColumn() - pastWeekTotalUniquePageviews: number - - @ViewColumn() - pastMonthTotalUniquePageviews: number - - @ViewColumn() - pastYearTotalUniquePageviews: number - - @ViewColumn() - pastAllTimeTotalUniquePageviews: number - @ViewColumn() pubDate?: Date diff --git a/src/entities/index.ts b/src/entities/index.ts index ade6d7c0..cf9a943e 100644 --- a/src/entities/index.ts +++ b/src/entities/index.ts @@ -17,6 +17,9 @@ export { Playlist } from './playlist' export { Podcast } from './podcast' export { RecentEpisodeByCategory } from './recentEpisodeByCategory' export { RecentEpisodeByPodcast } from './recentEpisodeByPodcast' +export { StatsEpisode } from './statsEpisode' +export { StatsMediaRef } from './statsMediaRef' +export { StatsPodcast } from './statsPodcast' export { UPDevice } from './upDevice' export { User } from './user' export { UserHistoryItem } from './userHistoryItem' diff --git a/src/entities/mediaRef.ts b/src/entities/mediaRef.ts index e5c606a2..fd5bb5f7 100644 --- a/src/entities/mediaRef.ts +++ b/src/entities/mediaRef.ts @@ -9,15 +9,26 @@ import { Entity, Generated, Index, + JoinColumn, JoinTable, ManyToMany, ManyToOne, OneToMany, + OneToOne, PrimaryColumn, Unique, UpdateDateColumn } from 'typeorm' -import { Author, Category, Episode, User, UserHistoryItem, UserNowPlayingItem, UserQueueItem } from '~/entities' +import { + Author, + Category, + Episode, + StatsMediaRef, + User, + UserHistoryItem, + UserNowPlayingItem, + UserQueueItem +} from '~/entities' import { generateShortId } from '~/lib/utility' @Entity('mediaRefs') @@ -80,48 +91,6 @@ export class MediaRef { @Column({ nullable: true }) linkUrl?: string - @Index() - @ValidateIf((a) => a.pastHourTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastHourTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastDayTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastWeekTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastMonthTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastYearTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastAllTimeTotalUniquePageviews: number - @Index() @IsInt() @Min(0) @@ -161,6 +130,10 @@ export class MediaRef { @OneToMany((type) => UserQueueItem, (userQueueItem) => userQueueItem.mediaRef) userQueueItems: UserQueueItem[] + @OneToOne(() => StatsMediaRef, { nullable: true }) + @JoinColumn({ name: 'stats_media_ref_id' }) + stats_media_ref?: StatsMediaRef + @CreateDateColumn() createdAt: Date diff --git a/src/entities/mediaRef_videos.ts b/src/entities/mediaRef_videos.ts index 6dcdb94a..8f0a5808 100644 --- a/src/entities/mediaRef_videos.ts +++ b/src/entities/mediaRef_videos.ts @@ -29,24 +29,6 @@ export class MediaRefVideos { @ViewColumn() linkUrl?: string - @ViewColumn() - pastHourTotalUniquePageviews: number - - @ViewColumn() - pastDayTotalUniquePageviews: number - - @ViewColumn() - pastWeekTotalUniquePageviews: number - - @ViewColumn() - pastMonthTotalUniquePageviews: number - - @ViewColumn() - pastYearTotalUniquePageviews: number - - @ViewColumn() - pastAllTimeTotalUniquePageviews: number - @ViewColumn() startTime: number diff --git a/src/entities/podcast.ts b/src/entities/podcast.ts index e2d4dab1..e8499ff2 100644 --- a/src/entities/podcast.ts +++ b/src/entities/podcast.ts @@ -1,8 +1,9 @@ +/* eslint-disable @typescript-eslint/camelcase */ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { IsUrl, IsInt, Min, ValidateIf } from 'class-validator' +import { IsUrl, ValidateIf } from 'class-validator' import { podcastItunesTypeDefaultValue, LiveItemStatus, PodcastMedium, ValueTagOriginal } from 'podverse-shared' -import { Author, Category, Episode, FeedUrl, Notification } from '~/entities' +import { Author, Category, Episode, FeedUrl, Notification, StatsPodcast } from '~/entities' import { BeforeInsert, BeforeUpdate, @@ -11,9 +12,11 @@ import { Entity, Generated, Index, + JoinColumn, JoinTable, ManyToMany, OneToMany, + OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' @@ -24,18 +27,6 @@ type Funding = { value?: string } -@Index(['hasVideo', 'pastAllTimeTotalUniquePageviews']) -@Index(['hasVideo', 'pastHourTotalUniquePageviews']) -@Index(['hasVideo', 'pastDayTotalUniquePageviews']) -@Index(['hasVideo', 'pastWeekTotalUniquePageviews']) -@Index(['hasVideo', 'pastMonthTotalUniquePageviews']) -@Index(['hasVideo', 'pastYearTotalUniquePageviews']) -@Index(['medium', 'pastAllTimeTotalUniquePageviews']) -@Index(['medium', 'pastHourTotalUniquePageviews']) -@Index(['medium', 'pastDayTotalUniquePageviews']) -@Index(['medium', 'pastWeekTotalUniquePageviews']) -@Index(['medium', 'pastMonthTotalUniquePageviews']) -@Index(['medium', 'pastYearTotalUniquePageviews']) @Entity('podcasts') export class Podcast { @PrimaryColumn('varchar', { @@ -53,12 +44,10 @@ export class Podcast { @Column({ nullable: true, unique: true }) podcastIndexId?: string - // This replaces the podcast.guid column @Index() @Column({ type: 'uuid', nullable: true }) podcastGuid?: string - // deprecated: use podcastGuid instead @Column({ nullable: true }) guid?: string @@ -147,9 +136,6 @@ export class Podcast { @Column({ nullable: true }) linkUrl?: string - // TODO: the Podcast.medium enum is currently missing the "mixed" value. - // It is also missing Medium Lists values (podcastL, musicL, etc.), - // but I don't know how those would fit into our UX yet. @Index() @Column({ type: 'enum', @@ -161,48 +147,6 @@ export class Podcast { @Column({ default: false }) parsingPriority?: boolean - @Index() - @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastAllTimeTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastHourTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastHourTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastDayTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastWeekTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastMonthTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastYearTotalUniquePageviews: number - @ValidateIf((a) => a.shrunkImageUrl != null) @IsUrl() @Column({ nullable: true }) @@ -246,6 +190,10 @@ export class Podcast { @OneToMany((type) => Notification, (notification) => notification.podcast) notifications: Notification[] + @OneToOne(() => StatsPodcast, { nullable: true }) + @JoinColumn({ name: 'stats_podcast_id' }) + stats_podcast?: StatsPodcast + @CreateDateColumn() createdAt: Date diff --git a/src/entities/statsEpisode.ts b/src/entities/statsEpisode.ts new file mode 100644 index 00000000..a379e7ef --- /dev/null +++ b/src/entities/statsEpisode.ts @@ -0,0 +1,56 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ + +import { IsInt, Min, ValidateIf } from 'class-validator' +import { Episode } from '~/entities' +import { Column, CreateDateColumn, Entity, Index, JoinColumn, OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' + +@Entity('stats_episode') +export class StatsEpisode { + @PrimaryColumn() + id: string + + @Index() + @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastDayTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastWeekTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastMonthTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastYearTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastAllTimeTotalUniquePageviews: number + + @OneToOne(() => Episode, { onDelete: 'CASCADE', nullable: false }) + @JoinColumn({ name: 'episode_id' }) + episode: Episode + + @CreateDateColumn() + createdAt: Date + + @UpdateDateColumn() + updatedAt: Date +} diff --git a/src/entities/statsMediaRef.ts b/src/entities/statsMediaRef.ts new file mode 100644 index 00000000..dcd1d424 --- /dev/null +++ b/src/entities/statsMediaRef.ts @@ -0,0 +1,56 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ + +import { IsInt, Min, ValidateIf } from 'class-validator' +import { MediaRef } from '~/entities' +import { Column, CreateDateColumn, Entity, Index, JoinColumn, OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' + +@Entity('stats_media_ref') +export class StatsMediaRef { + @PrimaryColumn() + id: string + + @Index() + @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastDayTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastWeekTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastMonthTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastYearTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastAllTimeTotalUniquePageviews: number + + @OneToOne(() => MediaRef, { onDelete: 'CASCADE', nullable: false }) + @JoinColumn({ name: 'media_ref_id' }) + mediaRef: MediaRef + + @CreateDateColumn() + createdAt: Date + + @UpdateDateColumn() + updatedAt: Date +} diff --git a/src/entities/statsPodcast.ts b/src/entities/statsPodcast.ts new file mode 100644 index 00000000..9b493d4c --- /dev/null +++ b/src/entities/statsPodcast.ts @@ -0,0 +1,56 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ + +import { IsInt, Min, ValidateIf } from 'class-validator' +import { Podcast } from '~/entities' +import { Column, CreateDateColumn, Entity, Index, JoinColumn, OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' + +@Entity('stats_podcast') +export class StatsPodcast { + @PrimaryColumn() + id: string + + @Index() + @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastDayTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastWeekTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastMonthTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastYearTotalUniquePageviews: number + + @Index() + @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) + @IsInt() + @Min(0) + @Column({ default: 0 }) + pastAllTimeTotalUniquePageviews: number + + @OneToOne(() => Podcast, { onDelete: 'CASCADE', nullable: false }) + @JoinColumn({ name: 'podcast_id' }) + podcast: Podcast + + @CreateDateColumn() + createdAt: Date + + @UpdateDateColumn() + updatedAt: Date +} diff --git a/src/lib/db.ts b/src/lib/db.ts index b774545a..65fcbf51 100644 --- a/src/lib/db.ts +++ b/src/lib/db.ts @@ -20,6 +20,9 @@ import { Podcast, RecentEpisodeByCategory, RecentEpisodeByPodcast, + StatsEpisode, + StatsMediaRef, + StatsPodcast, UPDevice, User, UserHistoryItem, @@ -48,6 +51,9 @@ const entities = [ Podcast, RecentEpisodeByCategory, RecentEpisodeByPodcast, + StatsEpisode, + StatsMediaRef, + StatsPodcast, UPDevice, User, UserHistoryItem, diff --git a/src/seeds/qa/stats.ts b/src/seeds/qa/stats.ts index e7d96d8e..cd7a5091 100644 --- a/src/seeds/qa/stats.ts +++ b/src/seeds/qa/stats.ts @@ -1,6 +1,6 @@ import { faker } from '@faker-js/faker' import { getRepository } from 'typeorm' -import { Episode, MediaRef, Podcast } from '~/entities' +import { Episode, MediaRef, Podcast, StatsEpisode, StatsMediaRef, StatsPodcast } from '~/entities' import { logPerformance, _logEnd, _logStart } from '~/lib/utility' const statsQAMinRange = 0 @@ -25,7 +25,6 @@ const statsQAGetNumber = () => { const statsQAGetPageviews = () => { return { - pastHourTotalUniquePageviews: statsQAGetNumber(), pastDayTotalUniquePageviews: statsQAGetNumber(), pastWeekTotalUniquePageviews: statsQAGetNumber(), pastMonthTotalUniquePageviews: statsQAGetNumber(), @@ -40,17 +39,16 @@ const statsQAUpdatePodcasts = async () => { const podcastRepository = getRepository(Podcast) const podcasts = await podcastRepository.find({ take: 1000 }) - const newPodcasts: any[] = [] + const statsPodcastsRepository = getRepository(StatsPodcast) for (const podcast of podcasts) { - const newPodcast = { - ...podcast, + const statsPodcast = await statsPodcastsRepository.find({ podcast }) + const newStatsPodcast = { + ...(statsPodcast?.[0] ? statsPodcast[0] : { podcast }), ...statsQAGetPageviews() } - newPodcasts.push(newPodcast) + await statsPodcastsRepository.save(newStatsPodcast) } - await podcastRepository.save(newPodcasts) - logPerformance('statsQAUpdatePodcasts', _logEnd) } @@ -60,17 +58,16 @@ const statsQAUpdateEpisodes = async () => { const episodeRepository = getRepository(Episode) const episodes = await episodeRepository.find({ take: 1000 }) - const newEpisodes: any[] = [] + const statsEpisodesRepository = getRepository(StatsEpisode) for (const episode of episodes) { - const newEpisode = { - ...episode, + const statsEpisode = await statsEpisodesRepository.find({ episode }) + const newStatsEpisode = { + ...(statsEpisode?.[0] ? statsEpisode[0] : { episode }), ...statsQAGetPageviews() } - newEpisodes.push(newEpisode) + await statsEpisodesRepository.save(newStatsEpisode) } - await episodeRepository.save(newEpisodes) - logPerformance('statsQAUpdateEpisodes', _logEnd) } @@ -80,16 +77,15 @@ const statsQAUpdateMediaRefs = async () => { const mediaRefRepository = getRepository(MediaRef) const mediaRefs = await mediaRefRepository.find({ take: 1000 }) - const newMediaRefs: any[] = [] + const statsMediaRefsRepository = getRepository(StatsMediaRef) for (const mediaRef of mediaRefs) { - const newMediaRef = { - ...mediaRef, + const statsMediaRef = await statsMediaRefsRepository.find({ mediaRef }) + const newStatsMediaRef = { + ...(statsMediaRef?.[0] ? statsMediaRef[0] : { mediaRef }), ...statsQAGetPageviews() } - newMediaRefs.push(newMediaRef) + await statsMediaRefsRepository.save(newStatsMediaRef) } - await mediaRefRepository.save(newMediaRefs) - logPerformance('statsQAUpdateMediaRefs', _logEnd) } diff --git a/src/services/stats.ts b/src/services/stats.ts index 9f6fff6b..38951b18 100644 --- a/src/services/stats.ts +++ b/src/services/stats.ts @@ -37,7 +37,6 @@ enum StartDateOffset { */ enum TimeRanges { - // hour = 'pastHourTotalUniquePageviews', day = 'pastDayTotalUniquePageviews', week = 'pastWeekTotalUniquePageviews', month = 'pastMonthTotalUniquePageviews', From ffb5e6579c9b1f0643965c358ba8187c3973151b Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Sun, 14 Apr 2024 17:41:28 -0500 Subject: [PATCH 02/16] Disable camelcase linter rule --- .eslintrc | 1 + 1 file changed, 1 insertion(+) diff --git a/.eslintrc b/.eslintrc index 8b04c236..16eb6fb0 100644 --- a/.eslintrc +++ b/.eslintrc @@ -7,6 +7,7 @@ "@typescript-eslint" ], "rules": { + "@typescript-eslint/camelcase": 0, "@typescript-eslint/explicit-function-return-type": 0, "@typescript-eslint/member-delimiter-style": 0, "@typescript-eslint/no-explicit-any": 0, From cbb20f716f7b3278e96cada5c899841e3223ca15 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Sun, 14 Apr 2024 17:42:51 -0500 Subject: [PATCH 03/16] Revise the stats tables --- migrations/0055_stats.sql | 75 +++++++++++++++-------------------- src/entities/statsEpisode.ts | 38 +++++------------- src/entities/statsMediaRef.ts | 38 +++++------------- src/entities/statsPodcast.ts | 38 +++++------------- src/lib/stats.ts | 3 ++ src/seeds/qa/stats.ts | 65 +++++++++++++++++------------- 6 files changed, 99 insertions(+), 158 deletions(-) create mode 100644 src/lib/stats.ts diff --git a/migrations/0055_stats.sql b/migrations/0055_stats.sql index f8071cb0..7a859381 100644 --- a/migrations/0055_stats.sql +++ b/migrations/0055_stats.sql @@ -1,65 +1,54 @@ + +-- Timeframe enum +CREATE TYPE timeframe_enum AS ENUM ('daily', 'weekly', 'monthly', 'yearly', 'all_time'); + -- Podcasts Stats CREATE TABLE IF NOT EXISTS stats_podcast ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - "pastDayTotalUniquePageviews" INTEGER DEFAULT 0, - "pastWeekTotalUniquePageviews" INTEGER DEFAULT 0, - "pastMonthTotalUniquePageviews" INTEGER DEFAULT 0, - "pastYearTotalUniquePageviews" INTEGER DEFAULT 0, - "pastAllTimeTotalUniquePageviews" INTEGER DEFAULT 0, - podcast_id VARCHAR NOT NULL UNIQUE REFERENCES podcasts(id) ON DELETE CASCADE, + id SERIAL PRIMARY KEY, + play_count INTEGER DEFAULT 0, + timeframe timeframe_enum NOT NULL, + podcast_id VARCHAR NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE, + CONSTRAINT unique_timeframe_podcast UNIQUE (timeframe, podcast_id), "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); -CREATE INDEX "stats_podcast_pastDayTotalUniquePageviews_idx" ON stats_podcast ("pastDayTotalUniquePageviews"); -CREATE INDEX "stats_podcast_pastWeekTotalUniquePageviews_idx" ON stats_podcast ("pastWeekTotalUniquePageviews"); -CREATE INDEX "stats_podcast_pastMonthTotalUniquePageviews_idx" ON stats_podcast ("pastMonthTotalUniquePageviews"); -CREATE INDEX "stats_podcast_pastYearTotalUniquePageviews_idx" ON stats_podcast ("pastYearTotalUniquePageviews"); -CREATE INDEX "stats_podcast_pastAllTimeTotalUniquePageviews_idx" ON stats_podcast ("pastAllTimeTotalUniquePageviews"); +CREATE INDEX "stats_podcast_play_count_idx" ON stats_podcast (play_count); +CREATE INDEX "stats_podcast_timeframe_idx" ON stats_podcast (timeframe); +CREATE INDEX "stats_podcast_podcast_id_idx" ON stats_podcast (podcast_id); -ALTER TABLE podcasts -ADD COLUMN stats_podcast_id VARCHAR UNIQUE; +ALTER TABLE podcasts ADD COLUMN stats_podcast_id VARCHAR UNIQUE; -- Episodes Stats CREATE TABLE IF NOT EXISTS stats_episode ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - "pastDayTotalUniquePageviews" INTEGER DEFAULT 0, - "pastWeekTotalUniquePageviews" INTEGER DEFAULT 0, - "pastMonthTotalUniquePageviews" INTEGER DEFAULT 0, - "pastYearTotalUniquePageviews" INTEGER DEFAULT 0, - "pastAllTimeTotalUniquePageviews" INTEGER DEFAULT 0, - episode_id VARCHAR NOT NULL UNIQUE REFERENCES episodes(id) ON DELETE CASCADE, + id SERIAL PRIMARY KEY, + play_count INTEGER DEFAULT 0, + timeframe timeframe_enum NOT NULL, + episode_id VARCHAR NOT NULL REFERENCES episodes(id) ON DELETE CASCADE, + CONSTRAINT unique_timeframe_episode UNIQUE (timeframe, episode_id), "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); -CREATE INDEX "stats_episode_pastDayTotalUniquePageviews_idx" ON stats_episode ("pastDayTotalUniquePageviews"); -CREATE INDEX "stats_episode_pastWeekTotalUniquePageviews_idx" ON stats_episode ("pastWeekTotalUniquePageviews"); -CREATE INDEX "stats_episode_pastMonthTotalUniquePageviews_idx" ON stats_episode ("pastMonthTotalUniquePageviews"); -CREATE INDEX "stats_episode_pastYearTotalUniquePageviews_idx" ON stats_episode ("pastYearTotalUniquePageviews"); -CREATE INDEX "stats_episode_pastAllTimeTotalUniquePageviews_idx" ON stats_episode ("pastAllTimeTotalUniquePageviews"); +CREATE INDEX "stats_episode_play_count_idx" ON stats_episode (play_count); +CREATE INDEX "stats_episode_timeframe_idx" ON stats_episode (timeframe); +CREATE INDEX "stats_episode_episode_id_idx" ON stats_episode (episode_id); -ALTER TABLE episodes -ADD COLUMN stats_episode_id VARCHAR UNIQUE; +ALTER TABLE episodes ADD COLUMN stats_episode_id VARCHAR UNIQUE; --- Media Refs Stats +-- MediaRef Stats CREATE TABLE IF NOT EXISTS stats_media_ref ( - id UUID PRIMARY KEY DEFAULT uuid_generate_v4(), - "pastDayTotalUniquePageviews" INTEGER DEFAULT 0, - "pastWeekTotalUniquePageviews" INTEGER DEFAULT 0, - "pastMonthTotalUniquePageviews" INTEGER DEFAULT 0, - "pastYearTotalUniquePageviews" INTEGER DEFAULT 0, - "pastAllTimeTotalUniquePageviews" INTEGER DEFAULT 0, - media_ref_id VARCHAR NOT NULL UNIQUE REFERENCES "mediaRefs"(id) ON DELETE CASCADE, + id SERIAL PRIMARY KEY, + play_count INTEGER DEFAULT 0, + timeframe timeframe_enum NOT NULL, + media_ref_id VARCHAR NOT NULL REFERENCES "mediaRefs"(id) ON DELETE CASCADE, + CONSTRAINT unique_timeframe_media_ref UNIQUE (timeframe, media_ref_id), "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP ); -CREATE INDEX "stats_media_ref_pastDayTotalUniquePageviews_idx" ON stats_media_ref ("pastDayTotalUniquePageviews"); -CREATE INDEX "stats_media_ref_pastWeekTotalUniquePageviews_idx" ON stats_media_ref ("pastWeekTotalUniquePageviews"); -CREATE INDEX "stats_media_ref_pastMonthTotalUniquePageviews_idx" ON stats_media_ref ("pastMonthTotalUniquePageviews"); -CREATE INDEX "stats_media_ref_pastYearTotalUniquePageviews_idx" ON stats_media_ref ("pastYearTotalUniquePageviews"); -CREATE INDEX "stats_media_ref_pastAllTimeTotalUniquePageviews_idx" ON stats_media_ref ("pastAllTimeTotalUniquePageviews"); +CREATE INDEX "stats_media_ref_play_count_idx" ON stats_media_ref (play_count); +CREATE INDEX "stats_media_ref_timeframe_idx" ON stats_media_ref (timeframe); +CREATE INDEX "stats_media_ref_media_ref_id_idx" ON stats_media_ref (media_ref_id); -ALTER TABLE "mediaRefs" -ADD COLUMN stats_media_ref_id VARCHAR UNIQUE; +ALTER TABLE "mediaRefs" ADD COLUMN stats_media_ref_id VARCHAR UNIQUE; diff --git a/src/entities/statsEpisode.ts b/src/entities/statsEpisode.ts index a379e7ef..be9f2411 100644 --- a/src/entities/statsEpisode.ts +++ b/src/entities/statsEpisode.ts @@ -1,8 +1,7 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ - import { IsInt, Min, ValidateIf } from 'class-validator' import { Episode } from '~/entities' import { Column, CreateDateColumn, Entity, Index, JoinColumn, OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' +import { StatsTimeFrames, timeframeEnumValues } from '~/lib/stats' @Entity('stats_episode') export class StatsEpisode { @@ -10,40 +9,21 @@ export class StatsEpisode { id: string @Index() - @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastDayTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastWeekTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) + @ValidateIf((a) => a.play_count != null) @IsInt() @Min(0) @Column({ default: 0 }) - pastMonthTotalUniquePageviews: number + play_count: number @Index() - @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastYearTotalUniquePageviews: number + @Column({ + type: 'enum', + enum: timeframeEnumValues, + nullable: false + }) + timeframe: StatsTimeFrames @Index() - @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastAllTimeTotalUniquePageviews: number - @OneToOne(() => Episode, { onDelete: 'CASCADE', nullable: false }) @JoinColumn({ name: 'episode_id' }) episode: Episode diff --git a/src/entities/statsMediaRef.ts b/src/entities/statsMediaRef.ts index dcd1d424..765a8356 100644 --- a/src/entities/statsMediaRef.ts +++ b/src/entities/statsMediaRef.ts @@ -1,8 +1,7 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ - import { IsInt, Min, ValidateIf } from 'class-validator' import { MediaRef } from '~/entities' import { Column, CreateDateColumn, Entity, Index, JoinColumn, OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' +import { StatsTimeFrames, timeframeEnumValues } from '~/lib/stats' @Entity('stats_media_ref') export class StatsMediaRef { @@ -10,40 +9,21 @@ export class StatsMediaRef { id: string @Index() - @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastDayTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastWeekTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) + @ValidateIf((a) => a.play_count != null) @IsInt() @Min(0) @Column({ default: 0 }) - pastMonthTotalUniquePageviews: number + play_count: number @Index() - @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastYearTotalUniquePageviews: number + @Column({ + type: 'enum', + enum: timeframeEnumValues, + nullable: false + }) + timeframe: StatsTimeFrames @Index() - @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastAllTimeTotalUniquePageviews: number - @OneToOne(() => MediaRef, { onDelete: 'CASCADE', nullable: false }) @JoinColumn({ name: 'media_ref_id' }) mediaRef: MediaRef diff --git a/src/entities/statsPodcast.ts b/src/entities/statsPodcast.ts index 9b493d4c..cfd9d2b3 100644 --- a/src/entities/statsPodcast.ts +++ b/src/entities/statsPodcast.ts @@ -1,8 +1,7 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ - import { IsInt, Min, ValidateIf } from 'class-validator' import { Podcast } from '~/entities' import { Column, CreateDateColumn, Entity, Index, JoinColumn, OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' +import { StatsTimeFrames, timeframeEnumValues } from '~/lib/stats' @Entity('stats_podcast') export class StatsPodcast { @@ -10,40 +9,21 @@ export class StatsPodcast { id: string @Index() - @ValidateIf((a) => a.pastDayTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastDayTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastWeekTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastWeekTotalUniquePageviews: number - - @Index() - @ValidateIf((a) => a.pastMonthTotalUniquePageviews != null) + @ValidateIf((a) => a.play_count != null) @IsInt() @Min(0) @Column({ default: 0 }) - pastMonthTotalUniquePageviews: number + play_count: number @Index() - @ValidateIf((a) => a.pastYearTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastYearTotalUniquePageviews: number + @Column({ + type: 'enum', + enum: timeframeEnumValues, + nullable: false + }) + timeframe: StatsTimeFrames @Index() - @ValidateIf((a) => a.pastAllTimeTotalUniquePageviews != null) - @IsInt() - @Min(0) - @Column({ default: 0 }) - pastAllTimeTotalUniquePageviews: number - @OneToOne(() => Podcast, { onDelete: 'CASCADE', nullable: false }) @JoinColumn({ name: 'podcast_id' }) podcast: Podcast diff --git a/src/lib/stats.ts b/src/lib/stats.ts new file mode 100644 index 00000000..7f24a3cd --- /dev/null +++ b/src/lib/stats.ts @@ -0,0 +1,3 @@ +export const timeframeEnumValues = ['daily', 'weekly', 'monthly', 'yearly', 'all_time'] as StatsTimeFrames[] + +export type StatsTimeFrames = 'daily' | 'weekly' | 'monthly' | 'yearly' | 'all_time' diff --git a/src/seeds/qa/stats.ts b/src/seeds/qa/stats.ts index cd7a5091..1300bcd6 100644 --- a/src/seeds/qa/stats.ts +++ b/src/seeds/qa/stats.ts @@ -1,6 +1,7 @@ import { faker } from '@faker-js/faker' import { getRepository } from 'typeorm' import { Episode, MediaRef, Podcast, StatsEpisode, StatsMediaRef, StatsPodcast } from '~/entities' +import { timeframeEnumValues } from '~/lib/stats' import { logPerformance, _logEnd, _logStart } from '~/lib/utility' const statsQAMinRange = 0 @@ -23,16 +24,6 @@ const statsQAGetNumber = () => { }) } -const statsQAGetPageviews = () => { - return { - pastDayTotalUniquePageviews: statsQAGetNumber(), - pastWeekTotalUniquePageviews: statsQAGetNumber(), - pastMonthTotalUniquePageviews: statsQAGetNumber(), - pastYearTotalUniquePageviews: statsQAGetNumber(), - pastAllTimeTotalUniquePageviews: statsQAGetNumber() - } -} - const statsQAUpdatePodcasts = async () => { logPerformance('statsQAUpdatePodcasts', _logStart) @@ -40,13 +31,19 @@ const statsQAUpdatePodcasts = async () => { const podcasts = await podcastRepository.find({ take: 1000 }) const statsPodcastsRepository = getRepository(StatsPodcast) - for (const podcast of podcasts) { - const statsPodcast = await statsPodcastsRepository.find({ podcast }) - const newStatsPodcast = { - ...(statsPodcast?.[0] ? statsPodcast[0] : { podcast }), - ...statsQAGetPageviews() + for (const timeframe of timeframeEnumValues) { + for (const podcast of podcasts) { + const statsPodcast = await statsPodcastsRepository.find({ + podcast, + timeframe + }) + const newStatsPodcast = { + ...(statsPodcast?.[0] ? statsPodcast[0] : { podcast }), + timeframe, + play_count: statsQAGetNumber() + } + await statsPodcastsRepository.save(newStatsPodcast) } - await statsPodcastsRepository.save(newStatsPodcast) } logPerformance('statsQAUpdatePodcasts', _logEnd) @@ -59,13 +56,19 @@ const statsQAUpdateEpisodes = async () => { const episodes = await episodeRepository.find({ take: 1000 }) const statsEpisodesRepository = getRepository(StatsEpisode) - for (const episode of episodes) { - const statsEpisode = await statsEpisodesRepository.find({ episode }) - const newStatsEpisode = { - ...(statsEpisode?.[0] ? statsEpisode[0] : { episode }), - ...statsQAGetPageviews() + for (const timeframe of timeframeEnumValues) { + for (const episode of episodes) { + const statsEpisode = await statsEpisodesRepository.find({ + episode, + timeframe + }) + const newStatsEpisode = { + ...(statsEpisode?.[0] ? statsEpisode[0] : { episode }), + timeframe, + play_count: statsQAGetNumber() + } + await statsEpisodesRepository.save(newStatsEpisode) } - await statsEpisodesRepository.save(newStatsEpisode) } logPerformance('statsQAUpdateEpisodes', _logEnd) @@ -78,13 +81,19 @@ const statsQAUpdateMediaRefs = async () => { const mediaRefs = await mediaRefRepository.find({ take: 1000 }) const statsMediaRefsRepository = getRepository(StatsMediaRef) - for (const mediaRef of mediaRefs) { - const statsMediaRef = await statsMediaRefsRepository.find({ mediaRef }) - const newStatsMediaRef = { - ...(statsMediaRef?.[0] ? statsMediaRef[0] : { mediaRef }), - ...statsQAGetPageviews() + for (const timeframe of timeframeEnumValues) { + for (const mediaRef of mediaRefs) { + const statsMediaRef = await statsMediaRefsRepository.find({ + mediaRef, + timeframe + }) + const newStatsMediaRef = { + ...(statsMediaRef?.[0] ? statsMediaRef[0] : { mediaRef }), + timeframe, + play_count: statsQAGetNumber() + } + await statsMediaRefsRepository.save(newStatsMediaRef) } - await statsMediaRefsRepository.save(newStatsMediaRef) } logPerformance('statsQAUpdateMediaRefs', _logEnd) From d3672bf7663cf589effc8752f62127b578cb1940 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Sun, 14 Apr 2024 19:10:05 -0500 Subject: [PATCH 04/16] Change OneToOne to OneToMany for stats tables --- migrations/0055_stats.sql | 6 ------ src/entities/episode.ts | 4 +--- src/entities/mediaRef.ts | 5 +---- src/entities/podcast.ts | 5 +---- 4 files changed, 3 insertions(+), 17 deletions(-) diff --git a/migrations/0055_stats.sql b/migrations/0055_stats.sql index 7a859381..8c7f8634 100644 --- a/migrations/0055_stats.sql +++ b/migrations/0055_stats.sql @@ -17,8 +17,6 @@ CREATE INDEX "stats_podcast_play_count_idx" ON stats_podcast (play_count); CREATE INDEX "stats_podcast_timeframe_idx" ON stats_podcast (timeframe); CREATE INDEX "stats_podcast_podcast_id_idx" ON stats_podcast (podcast_id); -ALTER TABLE podcasts ADD COLUMN stats_podcast_id VARCHAR UNIQUE; - -- Episodes Stats CREATE TABLE IF NOT EXISTS stats_episode ( id SERIAL PRIMARY KEY, @@ -34,8 +32,6 @@ CREATE INDEX "stats_episode_play_count_idx" ON stats_episode (play_count); CREATE INDEX "stats_episode_timeframe_idx" ON stats_episode (timeframe); CREATE INDEX "stats_episode_episode_id_idx" ON stats_episode (episode_id); -ALTER TABLE episodes ADD COLUMN stats_episode_id VARCHAR UNIQUE; - -- MediaRef Stats CREATE TABLE IF NOT EXISTS stats_media_ref ( id SERIAL PRIMARY KEY, @@ -50,5 +46,3 @@ CREATE TABLE IF NOT EXISTS stats_media_ref ( CREATE INDEX "stats_media_ref_play_count_idx" ON stats_media_ref (play_count); CREATE INDEX "stats_media_ref_timeframe_idx" ON stats_media_ref (timeframe); CREATE INDEX "stats_media_ref_media_ref_id_idx" ON stats_media_ref (media_ref_id); - -ALTER TABLE "mediaRefs" ADD COLUMN stats_media_ref_id VARCHAR UNIQUE; diff --git a/src/entities/episode.ts b/src/entities/episode.ts index 7e027fa4..896d5f01 100644 --- a/src/entities/episode.ts +++ b/src/entities/episode.ts @@ -27,7 +27,6 @@ import { CreateDateColumn, Entity, Index, - JoinColumn, JoinTable, ManyToMany, ManyToOne, @@ -191,8 +190,7 @@ export class Episode { @OneToMany((type) => UserQueueItem, (userQueueItem) => userQueueItem.episode) userQueueItems: UserQueueItem[] - @OneToOne(() => StatsEpisode, { nullable: true }) - @JoinColumn({ name: 'stats_episode_id' }) + @OneToMany(() => StatsEpisode, (stats_episode) => stats_episode.episode) stats_episode?: StatsEpisode @CreateDateColumn() diff --git a/src/entities/mediaRef.ts b/src/entities/mediaRef.ts index fd5bb5f7..67a20700 100644 --- a/src/entities/mediaRef.ts +++ b/src/entities/mediaRef.ts @@ -9,12 +9,10 @@ import { Entity, Generated, Index, - JoinColumn, JoinTable, ManyToMany, ManyToOne, OneToMany, - OneToOne, PrimaryColumn, Unique, UpdateDateColumn @@ -130,8 +128,7 @@ export class MediaRef { @OneToMany((type) => UserQueueItem, (userQueueItem) => userQueueItem.mediaRef) userQueueItems: UserQueueItem[] - @OneToOne(() => StatsMediaRef, { nullable: true }) - @JoinColumn({ name: 'stats_media_ref_id' }) + @OneToMany(() => StatsMediaRef, (stats_media_ref) => stats_media_ref.mediaRef) stats_media_ref?: StatsMediaRef @CreateDateColumn() diff --git a/src/entities/podcast.ts b/src/entities/podcast.ts index e8499ff2..2f446cfe 100644 --- a/src/entities/podcast.ts +++ b/src/entities/podcast.ts @@ -12,11 +12,9 @@ import { Entity, Generated, Index, - JoinColumn, JoinTable, ManyToMany, OneToMany, - OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' @@ -190,8 +188,7 @@ export class Podcast { @OneToMany((type) => Notification, (notification) => notification.podcast) notifications: Notification[] - @OneToOne(() => StatsPodcast, { nullable: true }) - @JoinColumn({ name: 'stats_podcast_id' }) + @OneToMany(() => StatsPodcast, (stats_podcast) => stats_podcast.podcast) stats_podcast?: StatsPodcast @CreateDateColumn() From f8b67c5cb35e18290489981f765dfe61761540fc Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Sun, 14 Apr 2024 19:10:33 -0500 Subject: [PATCH 05/16] Update popularity sorting in queries --- src/lib/utility/index.ts | 65 +++++++++++++++++++++++++++++++++------- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/src/lib/utility/index.ts b/src/lib/utility/index.ts index d623ea4b..92fa018a 100644 --- a/src/lib/utility/index.ts +++ b/src/lib/utility/index.ts @@ -1,5 +1,6 @@ export { validatePassword } from '~/lib/utility/validation' import { performance } from 'perf_hooks' +import { StatsEpisode, StatsMediaRef, StatsPodcast } from '~/entities' const shortid = require('shortid') export const delimitQueryValues = (ctx, keys) => { @@ -101,9 +102,7 @@ export const getManticoreOrderByColumnName = (sort) => { let orderByColumnName = '' let orderByDirection = 'desc' - if (sort === 'top-past-hour') { - orderByColumnName = 'pasthourtotaluniquepageviews' - } else if (sort === 'top-past-week') { + if (sort === 'top-past-week') { orderByColumnName = 'pastweektotaluniquepageviews' } else if (sort === 'top-past-month') { orderByColumnName = 'pastmonthtotaluniquepageviews' @@ -134,22 +133,61 @@ export const addOrderByToQuery = (qb, type, sort, sortDateKey, allowRandom, isFr const ascKey = 'ASC' const descKey = 'DESC' + let ormStatsType = null as any + if (type === 'podcast') { + ormStatsType = StatsPodcast + } else if (type === 'episode') { + ormStatsType = StatsEpisode + } else if (type === 'mediaRef') { + ormStatsType = StatsMediaRef + } + if (!sort && isFromManticoreSearch) { // apply no sorting } else if (sort === 'live-item-start-asc') { qb.orderBy(`liveItem.start`, ascKey) } else if (sort === 'live-item-start-desc') { qb.orderBy(`liveItem.start`, descKey) - } else if (sort === 'top-past-hour') { - qb.orderBy(`${type}.pastHourTotalUniquePageviews`, descKey) } else if (sort === 'top-past-day') { - qb.orderBy(`${type}.pastDayTotalUniquePageviews`, descKey) + qb.innerJoin( + ormStatsType, + `stats_${type}`, + `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + { timeframe: 'daily' } + ) + qb.orderBy(`stats_${type}.play_count`, descKey) + } else if (sort === 'top-past-week') { + qb.innerJoin( + ormStatsType, + `stats_${type}`, + `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + { timeframe: 'weekly' } + ) + qb.orderBy(`stats_${type}.play_count`, descKey) } else if (sort === 'top-past-month') { - qb.orderBy(`${type}.pastMonthTotalUniquePageviews`, descKey) + qb.innerJoin( + ormStatsType, + `stats_${type}`, + `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + { timeframe: 'monthly' } + ) + qb.orderBy(`stats_${type}.play_count`, descKey) } else if (sort === 'top-past-year') { - qb.orderBy(`${type}.pastYearTotalUniquePageviews`, descKey) + qb.innerJoin( + ormStatsType, + `stats_${type}`, + `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + { timeframe: 'yearly' } + ) + qb.orderBy(`stats_${type}.play_count`, descKey) } else if (sort === 'top-all-time') { - qb.orderBy(`${type}.pastAllTimeTotalUniquePageviews`, descKey) + qb.innerJoin( + ormStatsType, + `stats_${type}`, + `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + { timeframe: 'all_time' } + ) + qb.orderBy(`stats_${type}.play_count`, descKey) } else if (sort === 'most-recent') { qb.orderBy(`${type}.${sortDateKey}`, descKey) } else if (sort === 'oldest') { @@ -165,8 +203,13 @@ export const addOrderByToQuery = (qb, type, sort, sortDateKey, allowRandom, isFr } else if (sort === 'episode-number-asc') { qb.orderBy(`${type}.itunesEpisode`, ascKey) } else { - // sort = top-past-week - qb.orderBy(`${type}.pastWeekTotalUniquePageviews`, descKey) + qb.innerJoin( + ormStatsType, + `stats_${type}`, + `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + { timeframe: 'weekly' } + ) + qb.orderBy(`stats_${type}.play_count`, descKey) } return qb From ccbf6f5f187275fbbaa07f0324d5ea48cc22ee62 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Mon, 15 Apr 2024 00:28:46 -0500 Subject: [PATCH 06/16] Fix clips search where clause --- src/controllers/mediaRef.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/controllers/mediaRef.ts b/src/controllers/mediaRef.ts index 7109af32..33e06101 100644 --- a/src/controllers/mediaRef.ts +++ b/src/controllers/mediaRef.ts @@ -203,7 +203,7 @@ const getMediaRefs = async (query, isFromManticoreSearch?, totalOverride?) => { searchTitle: `%${searchTitle?.toLowerCase().trim()}%` }) - qb.andWhere('"mediaRef"."isOfficialChapter" IS null') + qb.andWhere('"mediaRef"."isOfficialChapter" IS false') if (mediaRefIds?.length) { qb.andWhere('mediaRef.id IN (:...mediaRefIds)', { mediaRefIds }) From 583d7ccce44850df9e3246dfd996137fe3dfd316 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Mon, 15 Apr 2024 17:26:03 -0500 Subject: [PATCH 07/16] Update stats service to use new stats tables --- migrations/0055_set_timestamps_trigger.sql | 10 + migrations/{0055_stats.sql => 0056_stats.sql} | 45 ++- src/services/stats.ts | 294 +++++++++--------- 3 files changed, 188 insertions(+), 161 deletions(-) create mode 100644 migrations/0055_set_timestamps_trigger.sql rename migrations/{0055_stats.sql => 0056_stats.sql} (55%) diff --git a/migrations/0055_set_timestamps_trigger.sql b/migrations/0055_set_timestamps_trigger.sql new file mode 100644 index 00000000..767c7c60 --- /dev/null +++ b/migrations/0055_set_timestamps_trigger.sql @@ -0,0 +1,10 @@ +-- Create the function to set timestamps with triggers + +CREATE FUNCTION set_timestamps() +RETURNS TRIGGER AS $$ +BEGIN + NEW."createdAt" := COALESCE(NEW."createdAt", NOW()); + NEW."updatedAt" := NOW(); + RETURN NEW; +END; +$$ LANGUAGE plpgsql; diff --git a/migrations/0055_stats.sql b/migrations/0056_stats.sql similarity index 55% rename from migrations/0055_stats.sql rename to migrations/0056_stats.sql index 8c7f8634..64f796b5 100644 --- a/migrations/0055_stats.sql +++ b/migrations/0056_stats.sql @@ -9,13 +9,24 @@ CREATE TABLE IF NOT EXISTS stats_podcast ( timeframe timeframe_enum NOT NULL, podcast_id VARCHAR NOT NULL REFERENCES podcasts(id) ON DELETE CASCADE, CONSTRAINT unique_timeframe_podcast UNIQUE (timeframe, podcast_id), - "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP + "createdAt" timestamp without time zone DEFAULT now() NOT NULL, + "updatedAt" timestamp without time zone DEFAULT now() NOT NULL ); CREATE INDEX "stats_podcast_play_count_idx" ON stats_podcast (play_count); CREATE INDEX "stats_podcast_timeframe_idx" ON stats_podcast (timeframe); CREATE INDEX "stats_podcast_podcast_id_idx" ON stats_podcast (podcast_id); +CREATE INDEX "stats_podcast_updated_at" on stats_podcast ("updatedAt"); + +CREATE TRIGGER set_timestamps_before_insert +BEFORE INSERT ON stats_podcast +FOR EACH ROW +EXECUTE FUNCTION set_timestamps(); + +CREATE TRIGGER set_timestamps_before_update +BEFORE UPDATE ON stats_podcast +FOR EACH ROW +EXECUTE FUNCTION set_timestamps(); -- Episodes Stats CREATE TABLE IF NOT EXISTS stats_episode ( @@ -24,13 +35,24 @@ CREATE TABLE IF NOT EXISTS stats_episode ( timeframe timeframe_enum NOT NULL, episode_id VARCHAR NOT NULL REFERENCES episodes(id) ON DELETE CASCADE, CONSTRAINT unique_timeframe_episode UNIQUE (timeframe, episode_id), - "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP + "createdAt" timestamp without time zone DEFAULT now() NOT NULL, + "updatedAt" timestamp without time zone DEFAULT now() NOT NULL ); CREATE INDEX "stats_episode_play_count_idx" ON stats_episode (play_count); CREATE INDEX "stats_episode_timeframe_idx" ON stats_episode (timeframe); CREATE INDEX "stats_episode_episode_id_idx" ON stats_episode (episode_id); +CREATE INDEX "stats_episode_updated_at" on stats_episode ("updatedAt"); + +CREATE TRIGGER set_timestamps_before_insert +BEFORE INSERT ON stats_episode +FOR EACH ROW +EXECUTE FUNCTION set_timestamps(); + +CREATE TRIGGER set_timestamps_before_update +BEFORE UPDATE ON stats_episode +FOR EACH ROW +EXECUTE FUNCTION set_timestamps(); -- MediaRef Stats CREATE TABLE IF NOT EXISTS stats_media_ref ( @@ -39,10 +61,21 @@ CREATE TABLE IF NOT EXISTS stats_media_ref ( timeframe timeframe_enum NOT NULL, media_ref_id VARCHAR NOT NULL REFERENCES "mediaRefs"(id) ON DELETE CASCADE, CONSTRAINT unique_timeframe_media_ref UNIQUE (timeframe, media_ref_id), - "createdAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP, - "updatedAt" TIMESTAMP DEFAULT CURRENT_TIMESTAMP + "createdAt" timestamp without time zone DEFAULT now() NOT NULL, + "updatedAt" timestamp without time zone DEFAULT now() NOT NULL ); CREATE INDEX "stats_media_ref_play_count_idx" ON stats_media_ref (play_count); CREATE INDEX "stats_media_ref_timeframe_idx" ON stats_media_ref (timeframe); CREATE INDEX "stats_media_ref_media_ref_id_idx" ON stats_media_ref (media_ref_id); +CREATE INDEX "stats_media_ref_updated_at" on stats_media_ref ("updatedAt"); + +CREATE TRIGGER set_timestamps_before_insert +BEFORE INSERT ON stats_media_ref +FOR EACH ROW +EXECUTE FUNCTION set_timestamps(); + +CREATE TRIGGER set_timestamps_before_update +BEFORE UPDATE ON stats_media_ref +FOR EACH ROW +EXECUTE FUNCTION set_timestamps(); diff --git a/src/services/stats.ts b/src/services/stats.ts index 38951b18..02130410 100644 --- a/src/services/stats.ts +++ b/src/services/stats.ts @@ -1,7 +1,7 @@ /* eslint-disable @typescript-eslint/camelcase */ import { getConnection } from 'typeorm' import { connectToDb } from '~/lib/db' -import { /* lastHour,*/ offsetDate } from '~/lib/utility' +import { _logEnd, _logStart, logPerformance, offsetDate } from '~/lib/utility' import { splitDateIntoEqualIntervals } from '~/lib/utility/date' import { queryMatomoData } from './matomo' const moment = require('moment') @@ -36,15 +36,28 @@ enum StartDateOffset { } */ -enum TimeRanges { - day = 'pastDayTotalUniquePageviews', - week = 'pastWeekTotalUniquePageviews', - month = 'pastMonthTotalUniquePageviews', - year = 'pastYearTotalUniquePageviews', - allTime = 'pastAllTimeTotalUniquePageviews' +enum QueryTimeFrame { + day = 'daily', + week = 'weekly', + month = 'monthly', + year = 'yearly', + allTime = 'all_time' +} + +enum QueryIntervals { + day = 1, + week = 4, + month = 10, + year = 100, + allTime = 200 } export const queryUniquePageviews = async (pagePath: string, timeRange) => { + logPerformance('queryUniquePageviews', _logStart) + /* This start time is later used to determine which rows can be reset to 0. */ + const jobStartTimeISO = new Date().toISOString() + console.log('jobStartTimeISO:', jobStartTimeISO) + const finalPagePath = PagePaths[pagePath] const startDateOffset = parseInt(StartDateOffset[timeRange], 10) @@ -54,7 +67,7 @@ export const queryUniquePageviews = async (pagePath: string, timeRange) => { return } - if (!Object.keys(TimeRanges).includes(timeRange)) { + if (!Object.keys(QueryTimeFrame).includes(timeRange)) { console.log('A valid timeRange must be provided in the second parameter.') console.log('Valid options are: day, week, month, year, allTime') return @@ -63,17 +76,21 @@ export const queryUniquePageviews = async (pagePath: string, timeRange) => { const startDate = new Date(timeRange === 'allTime' ? '2017-01-01' : offsetDate(startDateOffset)) const endDate = new Date(offsetDate()) - const numberOfIntervals = ['allTime'].includes(timeRange) ? 120 : ['year'].includes(timeRange) ? 24 : 1 + const numberOfIntervals = parseInt(QueryIntervals[timeRange], 10) const dateIntervals = splitDateIntoEqualIntervals(startDate, endDate, numberOfIntervals) let data: any[] = [] for (const dateInterval of dateIntervals) { + logPerformance('matomo stats query', _logStart) + console.log('query dateInterval query', dateInterval) const response: any = await queryMatomoData( moment(dateInterval.start).format('YYYY-MM-DD'), moment(dateInterval.end).format('YYYY-MM-DD'), finalPagePath ) data = data.concat(response.data) + console.log('data length:', data.length) + logPerformance('matomo stats query', _logEnd) } /* @@ -115,66 +132,88 @@ export const queryUniquePageviews = async (pagePath: string, timeRange) => { filteredData = filterCustomFeedUrls(data, videoLimit) } - await savePageviewsToDatabase(finalPagePath, timeRange, filteredData) + await savePageviewsToDatabase(finalPagePath, timeRange, filteredData, jobStartTimeISO) + logPerformance('queryUniquePageviews', _logEnd) } -const generateGetAllRelatedDataQueryString = (finalPagePath: string, timeRange) => { +const generateGetOutdatedDataQueryString = (finalPagePath: string, timeRange, jobStartTimeISO) => { let queryString = 'pagePath: string, timeRange, tableName: string' if (finalPagePath === PagePaths.podcasts) { queryString = ` - SELECT p.id, p."${TimeRanges[timeRange]}" + SELECT sp.id FROM "podcasts" p - WHERE p."${TimeRanges[timeRange]}">0 - AND p."hasVideo" IS FALSE - AND p."medium" = 'podcast'; + LEFT JOIN + stats_podcast sp ON p.id = sp.podcast_id + AND sp.timeframe = '${QueryTimeFrame[timeRange]}' + WHERE p."hasVideo" IS FALSE + AND p.medium = 'podcast' + AND sp."updatedAt" < '${jobStartTimeISO}' ` } else if (finalPagePath === PagePaths.episodes) { queryString = ` - SELECT e.id, e."${TimeRanges[timeRange]}" + SELECT se.id FROM "episodes" e JOIN "podcasts" p ON p.id = e."podcastId" - WHERE e."${TimeRanges[timeRange]}">0 - AND p."hasVideo" IS FALSE - AND p."medium" = 'podcast'; + LEFT JOIN + stats_episode se on e.id = se.episode_id + AND se.timeframe = '${QueryTimeFrame[timeRange]}' + WHERE p."hasVideo" IS FALSE + AND p."medium" = 'podcast' + AND se."updatedAt" < '${jobStartTimeISO}' ` } else if (finalPagePath === PagePaths.clips) { queryString = ` - SELECT id, "${TimeRanges[timeRange]}" - FROM "mediaRefs" - WHERE "${TimeRanges[timeRange]}">0 + SELECT smr.id + FROM "mediaRefs" mr + LEFT JOIN + stats_media_ref smr ON mr.id = smr.media_ref_id + AND smr.timeframe = '${QueryTimeFrame[timeRange]}' + WHERE smr."updatedAt" < '${jobStartTimeISO}' ` } else if (finalPagePath === PagePaths.albums) { queryString = ` - SELECT p.id, p."${TimeRanges[timeRange]}" + SELECT sp.id FROM "podcasts" p - WHERE p."${TimeRanges[timeRange]}">0 - AND p."hasVideo" IS FALSE - AND p."medium" = 'music'; + LEFT JOIN + stats_podcast sp ON p.id = sp.podcast_id + AND sp.timeframe = '${QueryTimeFrame[timeRange]}' + WHERE p."hasVideo" IS FALSE + AND p.medium = 'music' + AND sp."updatedAt" < '${jobStartTimeISO}' ` } else if (finalPagePath === PagePaths.tracks) { queryString = ` - SELECT e.id, e."${TimeRanges[timeRange]}" + SELECT se.id FROM "episodes" e JOIN "podcasts" p ON p.id = e."podcastId" - WHERE e."${TimeRanges[timeRange]}">0 - AND p."hasVideo" IS FALSE - AND p."medium" = 'music'; + LEFT JOIN + stats_episode se on e.id = se.episode_id + AND se.timeframe = '${QueryTimeFrame[timeRange]}' + WHERE p."hasVideo" IS FALSE + AND p."medium" = 'music' + AND se."updatedAt" < '${jobStartTimeISO}' ` } else if (finalPagePath === PagePaths.channels) { queryString = ` - SELECT p.id, p."${TimeRanges[timeRange]}" + SELECT sp.id FROM "podcasts" p - WHERE "${TimeRanges[timeRange]}">0 - AND p."hasVideo" IS TRUE; + LEFT JOIN + stats_podcast sp ON p.id = sp.podcast_id + AND sp.timeframe = '${QueryTimeFrame[timeRange]}' + WHERE p."hasVideo" IS TRUE + AND sp."updatedAt" < '${jobStartTimeISO}' ` } else if (finalPagePath === PagePaths.videos) { queryString = ` - SELECT e.id, e."${TimeRanges[timeRange]}" + SELECT se.id FROM "episodes" e JOIN "podcasts" p ON p.id = e."podcastId" - WHERE e."${TimeRanges[timeRange]}">0 - AND p."hasVideo" IS TRUE; + LEFT JOIN + stats_episode se on e.id = se.episode_id + AND se.timeframe = '${QueryTimeFrame[timeRange]}' + WHERE p."hasVideo" IS TRUE + AND se."updatedAt" < '${jobStartTimeISO}' ` } else { throw new Error('generateAllRelatedDataQueryString: Failed to generate queryString') @@ -188,74 +227,46 @@ const generateResetToZeroQueryString = (finalPagePath: string, timeRange, id: st if (finalPagePath === PagePaths.podcasts) { queryString = ` - UPDATE "podcasts" - SET "${TimeRanges[timeRange]}"=0 - WHERE id = '${id}' - AND "hasVideo" IS FALSE - AND "medium" = 'podcast'; + UPDATE "stats_podcast" + SET play_count=0 + WHERE id = '${id}'; ` } else if (finalPagePath === PagePaths.episodes) { queryString = ` - UPDATE "episodes" e - SET "${TimeRanges[timeRange]}" = 0 - WHERE id = '${id}' - AND "podcastId" - IN ( - SELECT p.id - FROM podcasts p - WHERE e."podcastId" = p.id - AND p."hasVideo" IS FALSE - AND p."medium" = 'podcast' - ); + UPDATE "stats_episode" + SET play_count=0 + WHERE id = '${id}'; ` } else if (finalPagePath === PagePaths.clips) { queryString = ` - UPDATE "mediaRefs" - SET "${TimeRanges[timeRange]}"=0 + UPDATE "stats_media_ref" + SET play_count=0 WHERE id = '${id}'; ` } else if (finalPagePath === PagePaths.albums) { queryString = ` - UPDATE "podcasts" - SET "${TimeRanges[timeRange]}"=0 - WHERE id = '${id}' - AND "hasVideo" IS FALSE - AND "medium" = 'music'; + UPDATE "stats_podcast" + SET play_count=0 + WHERE id = '${id}'; ` } else if (finalPagePath === PagePaths.tracks) { queryString = ` - UPDATE "episodes" AS e - SET "${TimeRanges[timeRange]}" = 0 - WHERE id = ${id} - AND "podcastId" - IN ( - SELECT p.id - FROM podcasts p - WHERE e."podcastId" = p.id - AND p."hasVideo" IS FALSE - AND p."medium" = 'music' - ); + UPDATE "stats_episode" + SET play_count=0 + WHERE id = '${id}'; ` } else if (finalPagePath === PagePaths.channels) { queryString = ` - UPDATE "podcasts" - SET "${TimeRanges[timeRange]}"=0 - WHERE id = '${id}' - AND "hasVideo" IS TRUE; + UPDATE "stats_podcast" + SET play_count=0 + WHERE id = '${id}'; ` } else if (finalPagePath === PagePaths.videos) { queryString = ` - UPDATE "episodes" AS e - SET "${TimeRanges[timeRange]}" = 0 - WHERE id = ${id} - AND "podcastId" - IN ( - SELECT p.id - FROM podcasts p - WHERE e."podcastId" = p.id - AND p."hasVideo" IS TRUE - ); - ` + UPDATE "stats_episode" + SET play_count=0 + WHERE id = '${id}'; + ` } else { throw new Error('generateAllRelatedDataQueryString: Failed to generate queryString') } @@ -268,73 +279,52 @@ const generateSetNewCountQuery = (finalPagePath: string, timeRange, id: string, if (finalPagePath === PagePaths.podcasts) { queryString = ` - UPDATE "podcasts" - SET "${TimeRanges[timeRange]}"=${sum_daily_nb_uniq_visitors} - WHERE id = '${id}' - AND "hasVideo" IS FALSE - AND "medium" = 'podcast'; + INSERT INTO stats_podcast (play_count, timeframe, podcast_id) + VALUES (${sum_daily_nb_uniq_visitors}, '${QueryTimeFrame[timeRange]}', '${id}') + ON CONFLICT (timeframe, podcast_id) + DO UPDATE SET play_count = EXCLUDED.play_count; ` } else if (finalPagePath === PagePaths.episodes) { queryString = ` - UPDATE "episodes" AS e - SET "${TimeRanges[timeRange]}"=${sum_daily_nb_uniq_visitors} - WHERE id = '${id}' - AND "podcastId" - IN ( - SELECT p.id - FROM podcasts p - WHERE e."podcastId" = p.id - AND p."hasVideo" IS FALSE - AND p."medium" = 'podcast' - ); + INSERT INTO stats_episode (play_count, timeframe, episode_id) + VALUES (${sum_daily_nb_uniq_visitors}, '${QueryTimeFrame[timeRange]}', '${id}') + ON CONFLICT (timeframe, episode_id) + DO UPDATE SET play_count = EXCLUDED.play_count; ` } else if (finalPagePath === PagePaths.clips) { queryString = ` - UPDATE "mediaRefs" - SET "${TimeRanges[timeRange]}"=${sum_daily_nb_uniq_visitors} - WHERE id = '${id}'; + INSERT INTO stats_media_ref (play_count, timeframe, media_ref_id) + VALUES (${sum_daily_nb_uniq_visitors}, '${QueryTimeFrame[timeRange]}', '${id}') + ON CONFLICT (timeframe, media_ref_id) + DO UPDATE SET play_count = EXCLUDED.play_count; ` } else if (finalPagePath === PagePaths.albums) { queryString = ` - UPDATE "podcasts" - SET "${TimeRanges[timeRange]}"=${sum_daily_nb_uniq_visitors} - WHERE id = '${id}' - AND "hasVideo" IS FALSE - AND "medium" = 'music'; + INSERT INTO stats_podcast (play_count, timeframe, podcast_id) + VALUES (${sum_daily_nb_uniq_visitors}, '${QueryTimeFrame[timeRange]}', '${id}') + ON CONFLICT (timeframe, podcast_id) + DO UPDATE SET play_count = EXCLUDED.play_count; ` } else if (finalPagePath === PagePaths.tracks) { queryString = ` - UPDATE "episodes" AS e - SET "${TimeRanges[timeRange]}"=${sum_daily_nb_uniq_visitors} - WHERE id = '${id}' - AND "podcastId" - IN ( - SELECT p.id - FROM podcasts p - WHERE e."podcastId" = p.id - AND p."hasVideo" IS FALSE - AND p."medium" = 'music' - ); + INSERT INTO stats_episode (play_count, timeframe, episode_id) + VALUES (${sum_daily_nb_uniq_visitors}, '${QueryTimeFrame[timeRange]}', '${id}') + ON CONFLICT (timeframe, episode_id) + DO UPDATE SET play_count = EXCLUDED.play_count; ` } else if (finalPagePath === PagePaths.channels) { queryString = ` - UPDATE "podcasts" - SET "${TimeRanges[timeRange]}"=${sum_daily_nb_uniq_visitors} - WHERE id = '${id}' - AND "hasVideo" IS TRUE; + INSERT INTO stats_podcast (play_count, timeframe, podcast_id) + VALUES (${sum_daily_nb_uniq_visitors}, '${QueryTimeFrame[timeRange]}', '${id}') + ON CONFLICT (timeframe, podcast_id) + DO UPDATE SET play_count = EXCLUDED.play_count; ` } else if (finalPagePath === PagePaths.videos) { queryString = ` - UPDATE "episodes" AS e - SET "${TimeRanges[timeRange]}"=${sum_daily_nb_uniq_visitors} - WHERE id = '${id}' - AND "podcastId" - IN ( - SELECT p.id - FROM podcasts p - WHERE e."podcastId" = p.id - AND p."hasVideo" IS TRUE - ); + INSERT INTO stats_episode (play_count, timeframe, episode_id) + VALUES (${sum_daily_nb_uniq_visitors}, '${QueryTimeFrame[timeRange]}', '${id}') + ON CONFLICT (timeframe, episode_id) + DO UPDATE SET play_count = EXCLUDED.play_count; ` } else { throw new Error('generateSetNewCountQuery: Failed to generate queryString') @@ -343,7 +333,7 @@ const generateSetNewCountQuery = (finalPagePath: string, timeRange, id: string, return queryString } -const savePageviewsToDatabase = async (finalPagePath: string, timeRange, data) => { +const savePageviewsToDatabase = async (finalPagePath: string, timeRange, data, jobStartTimeISO) => { await connectToDb() const matomoDataRows = data @@ -351,26 +341,7 @@ const savePageviewsToDatabase = async (finalPagePath: string, timeRange, data) = console.log('finalPagePath', finalPagePath) console.log('timeRange', timeRange) console.log('matomoDataRows.length', matomoDataRows.length) - console.log('TimeRange', TimeRanges[timeRange]) - - /* - The Matomo stats endpoint will only return data for pages that have a view in the past X days, - so we need to first set all of the table rows with values > 0 back to 0, - before writing the Matomo data to the table. - */ - - const getTableRowsWithStatsData = generateGetAllRelatedDataQueryString(finalPagePath, timeRange) - const tableRowsWithStatsData = await getConnection().createEntityManager().query(getTableRowsWithStatsData) - - for (const row of tableRowsWithStatsData) { - try { - const rawSQLUpdate = generateResetToZeroQueryString(finalPagePath, timeRange, row.id) - await getConnection().createEntityManager().query(rawSQLUpdate) - } catch (err) { - console.log('tableRowsWithStatsData err', err) - console.log('tableRowsWithStatsData err row', row) - } - } + console.log('TimeRange', QueryTimeFrame[timeRange]) for (const row of matomoDataRows) { try { @@ -397,4 +368,17 @@ const savePageviewsToDatabase = async (finalPagePath: string, timeRange, data) = console.log('row', row) } } + + const getOutdatedStatsRowsQuery = generateGetOutdatedDataQueryString(finalPagePath, timeRange, jobStartTimeISO) + const outdatedStatsRows = await getConnection().createEntityManager().query(getOutdatedStatsRowsQuery) + + for (const row of outdatedStatsRows) { + try { + const rawSQLUpdate = generateResetToZeroQueryString(finalPagePath, timeRange, row.id) + await getConnection().createEntityManager().query(rawSQLUpdate) + } catch (err) { + console.log('outdatedStatsRows err', err) + console.log('outdatedStatsRows err row', row) + } + } } From f1eab0d716120e20ed7c4f1e2274d634263233e2 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Mon, 15 Apr 2024 17:44:53 -0500 Subject: [PATCH 08/16] Create 0057_drop_old_stats.sql --- migrations/0057_drop_old_stats.sql | 135 +++++++++++++++++++++++++++++ 1 file changed, 135 insertions(+) create mode 100644 migrations/0057_drop_old_stats.sql diff --git a/migrations/0057_drop_old_stats.sql b/migrations/0057_drop_old_stats.sql new file mode 100644 index 00000000..89092006 --- /dev/null +++ b/migrations/0057_drop_old_stats.sql @@ -0,0 +1,135 @@ +-- +-- NOTE: THIS MUST ONLY BE RUN AFTER THE NEW API IS DEPLOYED. +-- + +-- Drop the podcasts stats indexes + +DO $$ +DECLARE + index_name TEXT; +BEGIN + FOR index_name IN + SELECT indexname + FROM pg_indexes + WHERE tablename = 'podcasts' + AND ( + indexdef ILIKE '%pastHourTotalUniquePageviews%' + OR indexdef ILIKE '%pastDayTotalUniquePageviews%' + OR indexdef ILIKE '%pastWeekTotalUniquePageviews%' + OR indexdef ILIKE '%pastMonthTotalUniquePageviews%' + OR indexdef ILIKE '%pastYearTotalUniquePageviews%' + OR indexdef ILIKE '%pastAllTimeTotalUniquePageviews%' + ) + LOOP + EXECUTE format('DROP INDEX IF EXISTS %I;', index_name); + END LOOP; +END $$; + +-- Drop the episodes stats indexes + +DO $$ +DECLARE + index_name TEXT; +BEGIN + FOR index_name IN + SELECT indexname + FROM pg_indexes + WHERE tablename = 'episodes' + AND ( + indexdef ILIKE '%pastHourTotalUniquePageviews%' + OR indexdef ILIKE '%pastDayTotalUniquePageviews%' + OR indexdef ILIKE '%pastWeekTotalUniquePageviews%' + OR indexdef ILIKE '%pastMonthTotalUniquePageviews%' + OR indexdef ILIKE '%pastYearTotalUniquePageviews%' + OR indexdef ILIKE '%pastAllTimeTotalUniquePageviews%' + ) + LOOP + EXECUTE format('DROP INDEX IF EXISTS %I;', index_name); + END LOOP; +END $$; + +-- Drop the mediaRefs stats indexes + +DO $$ +DECLARE + index_name TEXT; +BEGIN + FOR index_name IN + SELECT indexname + FROM pg_indexes + WHERE tablename = 'mediaRefs' + AND ( + indexdef ILIKE '%pastHourTotalUniquePageviews%' + OR indexdef ILIKE '%pastDayTotalUniquePageviews%' + OR indexdef ILIKE '%pastWeekTotalUniquePageviews%' + OR indexdef ILIKE '%pastMonthTotalUniquePageviews%' + OR indexdef ILIKE '%pastYearTotalUniquePageviews%' + OR indexdef ILIKE '%pastAllTimeTotalUniquePageviews%' + ) + LOOP + EXECUTE format('DROP INDEX IF EXISTS %I;', index_name); + END LOOP; +END $$; + +-- Drop old podcast stats columns + +ALTER TABLE podcasts +DROP COLUMN "pastHourTotalUniquePageviews"; + +ALTER TABLE podcasts +DROP COLUMN "pastDayTotalUniquePageviews"; + +ALTER TABLE podcasts +DROP COLUMN "pastWeekTotalUniquePageviews"; + +ALTER TABLE podcasts +DROP COLUMN "pastMonthTotalUniquePageviews"; + +ALTER TABLE podcasts +DROP COLUMN "pastYearTotalUniquePageviews"; + +ALTER TABLE podcasts +DROP COLUMN "pastAllTimeTotalUniquePageviews"; + +-- Drop old episode stats columns +-- First drop the dependent materialized view + +DROP MATERIALIZED VIEW "episodes_most_recent"; + +ALTER TABLE episodes +DROP COLUMN "pastHourTotalUniquePageviews"; + +ALTER TABLE episodes +DROP COLUMN "pastDayTotalUniquePageviews"; + +ALTER TABLE episodes +DROP COLUMN "pastWeekTotalUniquePageviews"; + +ALTER TABLE episodes +DROP COLUMN "pastMonthTotalUniquePageviews"; + +ALTER TABLE episodes +DROP COLUMN "pastYearTotalUniquePageviews"; + +ALTER TABLE episodes +DROP COLUMN "pastAllTimeTotalUniquePageviews"; + +-- Drop old mediaRefs stats columns + +ALTER TABLE "mediaRefs" +DROP COLUMN "pastHourTotalUniquePageviews"; + +ALTER TABLE "mediaRefs" +DROP COLUMN "pastDayTotalUniquePageviews"; + +ALTER TABLE "mediaRefs" +DROP COLUMN "pastWeekTotalUniquePageviews"; + +ALTER TABLE "mediaRefs" +DROP COLUMN "pastMonthTotalUniquePageviews"; + +ALTER TABLE "mediaRefs" +DROP COLUMN "pastYearTotalUniquePageviews"; + +ALTER TABLE "mediaRefs" +DROP COLUMN "pastAllTimeTotalUniquePageviews"; From 7f6552cb1290b90f9d983edca64719f63a3133db Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Wed, 17 Apr 2024 00:33:04 -0500 Subject: [PATCH 09/16] Update 0055_set_timestamps_trigger.sql --- migrations/0055_set_timestamps_trigger.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/migrations/0055_set_timestamps_trigger.sql b/migrations/0055_set_timestamps_trigger.sql index 767c7c60..702bbbea 100644 --- a/migrations/0055_set_timestamps_trigger.sql +++ b/migrations/0055_set_timestamps_trigger.sql @@ -3,7 +3,6 @@ CREATE FUNCTION set_timestamps() RETURNS TRIGGER AS $$ BEGIN - NEW."createdAt" := COALESCE(NEW."createdAt", NOW()); NEW."updatedAt" := NOW(); RETURN NEW; END; From 0c18f6f71b517d55bd4846dc39b9c28c5bf7ca1b Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Thu, 25 Apr 2024 14:48:22 -0500 Subject: [PATCH 10/16] Add drop mediaRefs_videos materialized view to 0057 migration --- migrations/0057_drop_old_stats.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/migrations/0057_drop_old_stats.sql b/migrations/0057_drop_old_stats.sql index 89092006..4aa17ff0 100644 --- a/migrations/0057_drop_old_stats.sql +++ b/migrations/0057_drop_old_stats.sql @@ -116,6 +116,8 @@ DROP COLUMN "pastAllTimeTotalUniquePageviews"; -- Drop old mediaRefs stats columns +DROP MATERIALIZED VIEW "mediaRefs_videos"; + ALTER TABLE "mediaRefs" DROP COLUMN "pastHourTotalUniquePageviews"; From adfcc622690bdb18e082c356bf928d6e27caffb9 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Fri, 26 Apr 2024 01:36:49 -0500 Subject: [PATCH 11/16] Change stats relationships on parent tables to OneToOne --- src/entities/episode.ts | 2 +- src/entities/mediaRef.ts | 3 ++- src/entities/podcast.ts | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/entities/episode.ts b/src/entities/episode.ts index 896d5f01..566525f5 100644 --- a/src/entities/episode.ts +++ b/src/entities/episode.ts @@ -190,7 +190,7 @@ export class Episode { @OneToMany((type) => UserQueueItem, (userQueueItem) => userQueueItem.episode) userQueueItems: UserQueueItem[] - @OneToMany(() => StatsEpisode, (stats_episode) => stats_episode.episode) + @OneToOne(() => StatsEpisode, (stats_episode) => stats_episode.episode) stats_episode?: StatsEpisode @CreateDateColumn() diff --git a/src/entities/mediaRef.ts b/src/entities/mediaRef.ts index 67a20700..30292826 100644 --- a/src/entities/mediaRef.ts +++ b/src/entities/mediaRef.ts @@ -13,6 +13,7 @@ import { ManyToMany, ManyToOne, OneToMany, + OneToOne, PrimaryColumn, Unique, UpdateDateColumn @@ -128,7 +129,7 @@ export class MediaRef { @OneToMany((type) => UserQueueItem, (userQueueItem) => userQueueItem.mediaRef) userQueueItems: UserQueueItem[] - @OneToMany(() => StatsMediaRef, (stats_media_ref) => stats_media_ref.mediaRef) + @OneToOne(() => StatsMediaRef, (stats_media_ref) => stats_media_ref.mediaRef) stats_media_ref?: StatsMediaRef @CreateDateColumn() diff --git a/src/entities/podcast.ts b/src/entities/podcast.ts index 2f446cfe..6263403c 100644 --- a/src/entities/podcast.ts +++ b/src/entities/podcast.ts @@ -15,6 +15,7 @@ import { JoinTable, ManyToMany, OneToMany, + OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm' @@ -188,7 +189,7 @@ export class Podcast { @OneToMany((type) => Notification, (notification) => notification.podcast) notifications: Notification[] - @OneToMany(() => StatsPodcast, (stats_podcast) => stats_podcast.podcast) + @OneToOne(() => StatsPodcast, (stats_podcast) => stats_podcast.podcast) stats_podcast?: StatsPodcast @CreateDateColumn() From 2a81a6230e49a522f4cb8427c043f683207026ff Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Fri, 26 Apr 2024 01:37:16 -0500 Subject: [PATCH 12/16] Fix order by queries when querying for mediaRefs --- src/lib/utility/index.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/lib/utility/index.ts b/src/lib/utility/index.ts index 92fa018a..c02bf9ec 100644 --- a/src/lib/utility/index.ts +++ b/src/lib/utility/index.ts @@ -133,6 +133,8 @@ export const addOrderByToQuery = (qb, type, sort, sortDateKey, allowRandom, isFr const ascKey = 'ASC' const descKey = 'DESC' + const statsParentId = type === 'mediaRef' ? 'media_ref' : type + let ormStatsType = null as any if (type === 'podcast') { ormStatsType = StatsPodcast @@ -152,7 +154,7 @@ export const addOrderByToQuery = (qb, type, sort, sortDateKey, allowRandom, isFr qb.innerJoin( ormStatsType, `stats_${type}`, - `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + `${type}.id = stats_${type}.${statsParentId}_id AND stats_${type}.timeframe = :timeframe`, { timeframe: 'daily' } ) qb.orderBy(`stats_${type}.play_count`, descKey) @@ -160,7 +162,7 @@ export const addOrderByToQuery = (qb, type, sort, sortDateKey, allowRandom, isFr qb.innerJoin( ormStatsType, `stats_${type}`, - `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + `${type}.id = stats_${type}.${statsParentId}_id AND stats_${type}.timeframe = :timeframe`, { timeframe: 'weekly' } ) qb.orderBy(`stats_${type}.play_count`, descKey) @@ -168,7 +170,7 @@ export const addOrderByToQuery = (qb, type, sort, sortDateKey, allowRandom, isFr qb.innerJoin( ormStatsType, `stats_${type}`, - `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + `${type}.id = stats_${type}.${statsParentId}_id AND stats_${type}.timeframe = :timeframe`, { timeframe: 'monthly' } ) qb.orderBy(`stats_${type}.play_count`, descKey) @@ -176,7 +178,7 @@ export const addOrderByToQuery = (qb, type, sort, sortDateKey, allowRandom, isFr qb.innerJoin( ormStatsType, `stats_${type}`, - `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + `${type}.id = stats_${type}.${statsParentId}_id AND stats_${type}.timeframe = :timeframe`, { timeframe: 'yearly' } ) qb.orderBy(`stats_${type}.play_count`, descKey) @@ -184,7 +186,7 @@ export const addOrderByToQuery = (qb, type, sort, sortDateKey, allowRandom, isFr qb.innerJoin( ormStatsType, `stats_${type}`, - `${type}.id = stats_${type}.${type}_id AND stats_${type}.timeframe = :timeframe`, + `${type}.id = stats_${type}.${statsParentId}_id AND stats_${type}.timeframe = :timeframe`, { timeframe: 'all_time' } ) qb.orderBy(`stats_${type}.play_count`, descKey) From 52250fd453b6ac945d4a31452ea105049cad76c6 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Fri, 26 Apr 2024 02:58:42 -0500 Subject: [PATCH 13/16] Create 0058_drop_episode_mediaUrl_index.sql --- migrations/0058_drop_episode_mediaUrl_index.sql | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 migrations/0058_drop_episode_mediaUrl_index.sql diff --git a/migrations/0058_drop_episode_mediaUrl_index.sql b/migrations/0058_drop_episode_mediaUrl_index.sql new file mode 100644 index 00000000..ae270f8b --- /dev/null +++ b/migrations/0058_drop_episode_mediaUrl_index.sql @@ -0,0 +1,4 @@ +-- Optionally could be done CONCURRENTLY +-- DROP INDEX CONCURRENTLY IDX_da6fc438d37c65927437b3107f; + +DROP INDEX IDX_da6fc438d37c65927437b3107f; From 4616697890f8fe9c38fe0b9adc454f91c9f97f97 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Fri, 26 Apr 2024 03:21:19 -0500 Subject: [PATCH 14/16] Update 0058_drop_episode_mediaUrl_index.sql --- migrations/0058_drop_episode_mediaUrl_index.sql | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/migrations/0058_drop_episode_mediaUrl_index.sql b/migrations/0058_drop_episode_mediaUrl_index.sql index ae270f8b..2ac2c3f0 100644 --- a/migrations/0058_drop_episode_mediaUrl_index.sql +++ b/migrations/0058_drop_episode_mediaUrl_index.sql @@ -1,4 +1,16 @@ -- Optionally could be done CONCURRENTLY --- DROP INDEX CONCURRENTLY IDX_da6fc438d37c65927437b3107f; -DROP INDEX IDX_da6fc438d37c65927437b3107f; +-- drop episodes.mediaUrl index +DROP INDEX "IDX_da6fc438d37c65927437b3107f"; + +-- drop episodes.title index +DROP INDEX "IDX_acd3fd6c4dff47ee1cd00ac582"; + +-- drop podcasts.title index +DROP INDEX "IDX_a65598c2450c4f601ecb341994"; + +-- drop podcasts.shrunkImageLastUpdated index +DROP INDEX "IDX_30403fff476188bfb18fd38f10"; + +--drop podcasts.feedLastUpdated index +DROP INDEX "IDX_09ae4505e3b4b2ddb27486187a"; From 30fa7513eac1c02b9dbff465e2ff73558834b609 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Fri, 26 Apr 2024 20:54:33 -0500 Subject: [PATCH 15/16] Limit podcast and episode description and subtitle to 4000 characters --- src/services/parser.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/services/parser.ts b/src/services/parser.ts index 0b19ce8d..37dca322 100644 --- a/src/services/parser.ts +++ b/src/services/parser.ts @@ -361,7 +361,7 @@ export const parseFeedUrl = async (feedUrl, forceReparsing = false, cacheBust = logPerformance('categoryRepo.save', _logEnd) podcast.categories = categories - podcast.description = meta.description + podcast.description = meta.description?.substring(0, 4000) podcast.feedLastParseFailed = false const feedLastUpdated = new Date(mostRecentUpdateDateFromFeed || meta.lastBuildDate || meta.pubDate || '') @@ -957,7 +957,7 @@ const assignParsedEpisodeData = async ( episode.chaptersUrl = parsedEpisode.chapters.url episode.chaptersType = parsedEpisode.chapters.type } - episode.description = parsedEpisode.summary || parsedEpisode.description + episode.description = (parsedEpisode.summary || parsedEpisode.description)?.substring(0, 4000) episode.duration = parsedEpisode.duration ? parseInt(parsedEpisode.duration, 10) : 0 /* TODO: podcast-partytime is missing type and funding on episode */ // episode.episodeType = parsedEpisode.type @@ -986,7 +986,7 @@ const assignParsedEpisodeData = async ( episode.socialInteraction = parsedEpisode.socialInteraction episode.soundbite = parsedEpisode.soundbite - episode.subtitle = parsedEpisode.subtitle + episode.subtitle = parsedEpisode.subtitle?.substring(0, 4000) episode.title = parsedEpisode.title episode.transcript = parsedEpisode.transcript episode.value = From 621d016e1c7886d617448841ded2d2a5aaf3f183 Mon Sep 17 00:00:00 2001 From: Mitch Downey Date: Fri, 26 Apr 2024 20:58:20 -0500 Subject: [PATCH 16/16] Rename migration files --- ...set_timestamps_trigger.sql => 0056_set_timestamps_trigger.sql} | 0 migrations/{0056_stats.sql => 0057_stats.sql} | 0 migrations/{0057_drop_old_stats.sql => 0058_drop_old_stats.sql} | 0 ...de_mediaUrl_index.sql => 0059_drop_episode_mediaUrl_index.sql} | 0 4 files changed, 0 insertions(+), 0 deletions(-) rename migrations/{0055_set_timestamps_trigger.sql => 0056_set_timestamps_trigger.sql} (100%) rename migrations/{0056_stats.sql => 0057_stats.sql} (100%) rename migrations/{0057_drop_old_stats.sql => 0058_drop_old_stats.sql} (100%) rename migrations/{0058_drop_episode_mediaUrl_index.sql => 0059_drop_episode_mediaUrl_index.sql} (100%) diff --git a/migrations/0055_set_timestamps_trigger.sql b/migrations/0056_set_timestamps_trigger.sql similarity index 100% rename from migrations/0055_set_timestamps_trigger.sql rename to migrations/0056_set_timestamps_trigger.sql diff --git a/migrations/0056_stats.sql b/migrations/0057_stats.sql similarity index 100% rename from migrations/0056_stats.sql rename to migrations/0057_stats.sql diff --git a/migrations/0057_drop_old_stats.sql b/migrations/0058_drop_old_stats.sql similarity index 100% rename from migrations/0057_drop_old_stats.sql rename to migrations/0058_drop_old_stats.sql diff --git a/migrations/0058_drop_episode_mediaUrl_index.sql b/migrations/0059_drop_episode_mediaUrl_index.sql similarity index 100% rename from migrations/0058_drop_episode_mediaUrl_index.sql rename to migrations/0059_drop_episode_mediaUrl_index.sql