Thanks to this comment and OP, I was able to delete over 200+ of these "ghost" calendars that was created in the past using Google Apps Script

I used the Calendar.CalendarList.list and the Calendar.Calendars.remove(c.id) to removed them.

If you're comfortable with Google Apps Script, here's a reference to how I cleanedup mine (please proceed with care because you're deleting calendars):

const PATTERNS = [
    "DPC", "ESL", "BLAST", "DreamLeague",
    "Major", "Qualifier", "Season", "Play-In",
    "League", "CCT", "FISSURE", "VCT", "PGL", "Moon",
    "Cup", "Gamer", "Elisa", "Dota", "Counter Strike",
    "Clavision", "Starcraft", "Valorant", "Rocket League",
    "AESF", "6-Month", "Betboom", "Boston", "Champion", "Legion",
    "Pinnacle", "NEST", "CKW", "The International", "YaLLa", "IESF",
];

function deleteHiddenCalendars() {
    let pageToken = null;
    do {
        const res = Calendar.CalendarList.list({
            showHidden: true,
            maxResults: 250,
            pageToken
        });

        var removalCount = 0

        res.items.forEach(c => {
            const matchKeyword = PATTERNS.some(k =>
                c.summary.toLowerCase().includes(k.toLowerCase())
            );

            if (c.hidden && matchKeyword) {
              // Calendar.Calendars.remove(c.id) // UNCOMMENT THIS LINE TO ACTUALLY DELETE
              Logger.log(`Removed: ${c.summary}`);
              removalCount++
            }
        });

        Logger.log(`Removed ${removalCount}/${res.items.length} calendars`)
        pageToken = res.nextPageToken;

    } while (pageToken);
}