Efficient_VIRTUOSO.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "name": "Efficient_VIRTUOSO.ipynb",
      "private_outputs": true,
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "machine_shape": "hm",
      "include_colab_link": true
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/asigalov61/Intelligent-VIRTUOSO/blob/main/Efficient_VIRTUOSO.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QA0W-VK1JVQl"
      },
      "source": [
        "# Efficient VIRTUOSO (ver. 2.0)\n",
        "\n",
        "## \"Music never allows falsehoods for even the deaf hear flat notes!\" ---EV\n",
        "\n",
        "***\n",
        "\n",
        "## Chordified GPT2-based Symbolic Music Artificial Intelligence Model Creator/Trainer.\n",
        "\n",
        "### Multi-Instrumental, with special TMIDI Processors\n",
        "\n",
        "***\n",
        "\n",
        "Credit for char-based GPT2 implementation used in this colab goes out to Andrej Karpathy: https://github.com/karpathy/minGPT\n",
        "\n",
        "***\n",
        "\n",
        "WARNING: This complete implementation is a functioning model of the Artificial Intelligence. Please excercise great humility, care, and respect.\n",
        "\n",
        "***\n",
        "\n",
        "#### Project Los Angeles\n",
        "\n",
        "#### Tegridy Code 2021\n",
        "\n",
        "***"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eftzIVKrqR5S"
      },
      "source": [
        "# Setup Environment, clone needed repos, and install all required dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HsUtsJGNz6f2",
        "cellView": "form"
      },
      "source": [
        "#@title Install all dependencies (run only once per session)\n",
        "\n",
        "!git clone https://github.com/asigalov61/tegridy-tools\n",
        "!apt install fluidsynth #Pip does not work for some reason. Only apt works\n",
        "!pip install midi2audio"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pf8B3p6QySmE",
        "cellView": "form"
      },
      "source": [
        "#@title Import all needed modules\n",
        "\n",
        "print('Loading needed modules. Please wait...')\n",
        "import os\n",
        "\n",
        "if not os.path.exists('/content/Dataset'):\n",
        "    os.makedirs('/content/Dataset')\n",
        "\n",
        "os.chdir('/content/tegridy-tools/tegridy-tools')\n",
        "import TMIDI\n",
        "\n",
        "os.chdir('/content/tegridy-tools/tegridy-tools/minGPT')\n",
        "from minGPT import *\n",
        "\n",
        "from midi2audio import FluidSynth\n",
        "from IPython.display import display, Javascript, HTML, Audio\n",
        "\n",
        "from google.colab import output, drive\n",
        "\n",
        "os.chdir('/content/')\n",
        "print('Loading complete. Enjoy! :)')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N2Pv5eNRqiyr"
      },
      "source": [
        "# Download and process MIDI dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EHuggjW7etzZ",
        "cellView": "form"
      },
      "source": [
        "#@title Download special Tegridy Piano MIDI dataset\n",
        "\n",
        "#@markdown Works best stand-alone/as-is for the optimal results\n",
        "%cd /content/Dataset/\n",
        "\n",
        "!wget 'https://github.com/asigalov61/Tegridy-MIDI-Dataset/raw/master/Tegridy-Piano-CC-BY-NC-SA.zip'\n",
        "!unzip -j '/content/Dataset/Tegridy-Piano-CC-BY-NC-SA.zip'\n",
        "!rm '/content/Dataset/Tegridy-Piano-CC-BY-NC-SA.zip'\n",
        "\n",
        "%cd /content/"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-1ypXZoySkHJ"
      },
      "source": [
        "# If you are not sure where to start or what settings to select, please use original defaults"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BRy0VfpFIpaj",
        "cellView": "form"
      },
      "source": [
        "#@title Process MIDIs to special MIDI dataset with Tegridy MIDI Processor\n",
        "#@markdown NOTES:\n",
        "\n",
        "#@markdown 1) Dataset MIDI file names are used as song names. Feel free to change it to anything you like.\n",
        "\n",
        "#@markdown 2) Best results are achieved with the single-track, single-channel, single-instrument MIDI 0 files with plain English names (avoid special or sys/foreign chars)\n",
        "\n",
        "#@markdown 3) MIDI Channel = -1 means all MIDI channels. MIDI Channel = 16 means all channels will be processed. Otherwise, only single indicated MIDI channel will be processed.\n",
        "\n",
        "file_name_to_output_dataset_to = \"/content/Efficient-Virtuoso-Music-MIDI-Dataset\" #@param {type:\"string\"}\n",
        "desired_MIDI_channel_to_process = 0 #@param {type:\"slider\", min:-1, max:15, step:1}\n",
        "MIDI_events_time_denominator = 10 #@param {type:\"slider\", min:1, max:100, step:1}\n",
        "melody_notes_in_chords = True #@param {type:\"boolean\"}\n",
        "\n",
        "print('TMIDI Processor')\n",
        "print('Starting up...')\n",
        "\n",
        "###########\n",
        "\n",
        "average_note_pitch = 0\n",
        "min_note = 127\n",
        "max_note = 0\n",
        "\n",
        "files_count = 0\n",
        "\n",
        "ev = 0\n",
        "\n",
        "chords_list_f = []\n",
        "melody_list_f = []\n",
        "\n",
        "chords_list = []\n",
        "chords_count = 0\n",
        "\n",
        "melody_chords = []\n",
        "melody_count = 0\n",
        "\n",
        "###########\n",
        "\n",
        "print('Loading MIDI files...')\n",
        "print('This may take a while on a large dataset in particular.')\n",
        "\n",
        "dataset_addr = \"/content/Dataset/\"\n",
        "os.chdir(dataset_addr)\n",
        "filez = os.listdir(dataset_addr)\n",
        "\n",
        "print('Processing MIDI files. Please wait...')\n",
        "for f in tqdm.auto.tqdm(filez):\n",
        "  try:\n",
        "    files_count += 1\n",
        "    chords_list, melody = TMIDI.Tegridy_MIDI_Processor(f, \n",
        "                                                      desired_MIDI_channel_to_process, \n",
        "                                                      MIDI_events_time_denominator,\n",
        "                                                      )\n",
        "\n",
        "    fn = os.path.basename(f)\n",
        "    fno = fn.split('.')[0].replace(' ', '_')\n",
        "\n",
        "    chords_l, melody_l = TMIDI.Tegridy_Chords_Converter(chords_list, \n",
        "                                                        melody, \n",
        "                                                        fno, \n",
        "                                                        melody_notes_in_chords)\n",
        "    \n",
        "    chords_list_f.extend(chords_l)\n",
        "    melody_list_f.extend(melody_l)\n",
        "    chords_count += len(chords_list)\n",
        "    melody_count += len(melody_l)\n",
        "  \n",
        "  except:\n",
        "    print('Bad MIDI:', f)\n",
        "    continue\n",
        "\n",
        "average_note_pitch = int((min_note + max_note) / 2)\n",
        "\n",
        "print('Task complete :)')\n",
        "print('==================================================')\n",
        "print('Number of processed dataset MIDI files:', files_count)\n",
        "print('Average note pitch =', average_note_pitch)\n",
        "#print('Min note pitch =', min_note)\n",
        "#print('Max note pitch =', max_note)\n",
        "#print('Number of MIDI events recorded:', len(events_matrix))\n",
        "print('Number of MIDI chords recorded:', chords_count)\n",
        "print('The longest chord:', len(max(chords_list_f, key=len)), 'notes') \n",
        "print(max(chords_list_f, key=len))\n",
        "print('Number of recorded melody events:', len(melody_list_f))\n",
        "print('First melody event:', melody_list_f[0], 'Last Melody event:', melody_list_f[-1])\n",
        "print('Total number of MIDI events recorded:', len(chords_list_f))\n",
        "\n",
        "# Dataset\n",
        "MusicDataset = [chords_list_f, melody_list_f]\n",
        "\n",
        "# Writing dataset to pickle file\n",
        "TMIDI.Tegridy_Pickle_File_Writer(MusicDataset, file_name_to_output_dataset_to)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A378EUz_7K0G",
        "cellView": "form"
      },
      "source": [
        "#@title Process MIDI Dataset to TXT Dataset (w/Tegridy MIDI-TXT Processor)\n",
        "full_path_to_TXT_dataset = \"/content/Efficient-Virtuoso-Music-TXT-Dataset.txt\" #@param {type:\"string\"}\n",
        "line_by_line_dataset = True #@param {type:\"boolean\"}\n",
        "chords_durations_multiplier = 1 #@param {type:\"slider\", min:0.1, max:2, step:0.1}\n",
        "simulate_velocity = True #@param {type:\"boolean\"}\n",
        "reduce_MIDI_channels = False #@param {type:\"boolean\"}\n",
        "reduce_notes_velocities = False #@param {type:\"boolean\"}\n",
        "\n",
        "# MIDI Dataset to txt dataset converter \n",
        "print('TMIDI-TXT Processor')\n",
        "print('Starting up...')\n",
        "\n",
        "if os.path.exists(full_path_to_TXT_dataset):\n",
        "  os.remove(full_path_to_TXT_dataset)\n",
        "  print('Removing old Dataset...')\n",
        "\n",
        "else:\n",
        "  print(\"Creating new Dataset file...\")\n",
        "\n",
        "if simulate_velocity:\n",
        "  print('Simulated velocity mode is enabled.')\n",
        "\n",
        "TXT = ''\n",
        "number_of_chords = 0\n",
        "number_of_bad_chords = 0\n",
        "dataset_name = 'DATASET=Intelligent_VIRTUOSO_TXT_Music_Dataset'\n",
        "\n",
        "file = open(full_path_to_TXT_dataset, 'a')\n",
        "TXT, number_of_chords, number_of_bad_chords = TMIDI.Tegridy_MIDI_TXT_Processor(dataset_name, \n",
        "                                                                               chords_list_f, \n",
        "                                                                               melody_list_f, \n",
        "                                                                               simulate_velocity, \n",
        "                                                                               line_by_line_dataset,\n",
        "                                                                               0,\n",
        "                                                                               chords_durations_multiplier,\n",
        "                                                                               )\n",
        "\n",
        "print('Number of chords recorded: ', number_of_chords)\n",
        "print('Number of bad/skipped chords: ', number_of_bad_chords)\n",
        "print('Done!')\n",
        "\n",
        "TXT1, n = TMIDI.Tegridy_TXT_Reducer(TXT, include_MIDI_channels=reduce_MIDI_channels, include_notes_velocities=reduce_notes_velocities)\n",
        "\n",
        "file.write(TXT1)\n",
        "file.close()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0cj2xl5xqwea"
      },
      "source": [
        "# Setup and Intialize the Model\r\n",
        "\r\n",
        "## YOU MUST RUN THE CELL/CODE IN THE SECTION BELOW to init the model. Does not matter if the model is empty or pre-trained.\r\n",
        "\r\n",
        "## DO NOT EXECUTE TRAIN CELL/CODE UNLESS YOU INTEND TO TRAIN FROM SCRATCH"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "m4QIgbe3ySmN",
        "cellView": "form"
      },
      "source": [
        "#@title Create/prepare GPT2 model and load the dataset\n",
        "\n",
        "full_path_to_training_text_file = \"/content/Efficient-Virtuoso-Music-TXT-Dataset.txt\" #@param {type:\"string\"}\n",
        "model_attention_span_in_tokens = 512 #@param {type:\"slider\", min:0, max:1024, step:16}\n",
        "model_embed_size = 256 #@param {type:\"slider\", min:0, max:1024, step:64}\n",
        "number_of_heads = 16 #@param {type:\"slider\", min:1, max:16, step:1}\n",
        "number_of_layers = 4 #@param {type:\"slider\", min:1, max:16, step:1}\n",
        "number_of_training_epochs = 2 #@param {type:\"slider\", min:1, max:5, step:1}\n",
        "training_batch_size = 48 #@param {type:\"slider\", min:0, max:160, step:4}\n",
        "model_learning_rate = 6e-4 #@param {type:\"number\"}\n",
        "\n",
        "trainer, model, train_dataset = MainLoader(full_path_to_training_text_file,\n",
        "                                          None,\n",
        "                                          16,\n",
        "                                          model_attention_span_in_tokens,\n",
        "                                          model_embed_size,\n",
        "                                          number_of_heads,\n",
        "                                          number_of_layers,\n",
        "                                          number_of_training_epochs,\n",
        "                                          training_batch_size,\n",
        "                                          model_learning_rate)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "B_18H-M-q4CB"
      },
      "source": [
        "# Train the model or Load/Re-load the existing pre-trained model checkpoint"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cRffqT9WFBHB",
        "cellView": "form"
      },
      "source": [
        "#@title (OPTION 1) Train the model\n",
        "%cd /content/\n",
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YVWEhUj1cg7N",
        "cellView": "form"
      },
      "source": [
        "#@title Plot Positional Embeddings\n",
        "\n",
        "# visualize some of the learned positional embeddings, maybe they contain structure\n",
        "PlotPositionalEmbeddings(model, model_attention_span_in_tokens)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMkyEMghC-KR",
        "cellView": "form"
      },
      "source": [
        "#@title Save/Re-Save the model from memory\n",
        "#@markdown Standard PyTorch AI models file extension is PTH\n",
        "full_path_to_save_model_to = \"/content/Efficient-Virtuoso-Trained-Model.pth\" #@param {type:\"string\"}\n",
        "%cd /content/\n",
        "torch.save(model, full_path_to_save_model_to)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CmD7VRZhDcnJ",
        "cellView": "form"
      },
      "source": [
        "#@title (OPTION 2) Load existing model/checkpoint\n",
        "full_path_to_model_checkpoint = \"/content/Efficient-Virtuoso-Trained-Model.pth\" #@param {type:\"string\"}\n",
        "model = torch.load(full_path_to_model_checkpoint)\n",
        "model.eval()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FfgeQl8_rEox"
      },
      "source": [
        "# Generate, download, plot, and listen to the output"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6ZEqKJ6NySmV",
        "cellView": "form"
      },
      "source": [
        "#@title Generate and download the composition as TXT file.\n",
        "#@markdown PLEASE NOTE IMPORTANT POINTS: \n",
        "\n",
        "#@markdown 0) If you are not sure where to start/what settings to set, please use original defaults.\n",
        "\n",
        "#@markdown 1) Model primes from the dataset !!!\n",
        "\n",
        "#@markdown 2) Model's first output may be empty or garbled so please try several times before discarting the model\n",
        "\n",
        "#@markdown 3) You can now communicate to the model desired length of the output composition by suffixing input_prompt with number of chords.\n",
        "\n",
        "#@markdown I.e. SONG=Relax_with_900_Chords\n",
        "\n",
        "#@markdown 3) Coherence of GPT2 Models is inversly proportional to the length of the generated composition, so the best resutls are achieved with shorter compositions and/or continuation routines use (which be implemented in the future version of Intelligent VIRTUOSO)\n",
        "\n",
        "print('Efficient VIRTUOSO Model Generator')\n",
        "print('Starting up...')\n",
        "number_of_tokens_to_generate = 2048 #@param {type:\"slider\", min:0, max:32768, step:128}\n",
        "creativity_temperature = 0.8 #@param {type:\"slider\", min:0.05, max:4, step:0.05}\n",
        "top_k_prob = 4 #@param {type:\"slider\", min:0, max:50, step:1}\n",
        "input_prompt = \"SONG=Relax\" #@param {type:\"string\"}\n",
        "\n",
        "os.chdir('/content/')\n",
        "\n",
        "completion = Generate(model,\n",
        "                      train_dataset,\n",
        "                      trainer,\n",
        "                      number_of_tokens_to_generate,\n",
        "                      creativity_temperature,\n",
        "                      top_k_prob,\n",
        "                      input_prompt)\n",
        "\n",
        "# Stuff for datetime stamp\n",
        "filename = '/content/Efficient-VIRTUOSO-Composition-' + 'generated-on-' \n",
        "fname = TMIDI.Tegridy_File_Time_Stamp(filename)\n",
        "\n",
        "print('Done!')\n",
        "print('Saving to', str(fname + '.txt'))\n",
        "with open(fname + '.txt', \"w\") as text_file:\n",
        "    print(completion, file=text_file)\n",
        "\n",
        "print('Downloading TXT file...')\n",
        "from google.colab import files\n",
        "files.download(fname + '.txt')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "22qDcd4NO2bs",
        "cellView": "form"
      },
      "source": [
        "#@title Convert to MIDI from TXT (w/Tegridy MIDI-TXT Processor)\n",
        "\n",
        "#@markdown Standard MIDI timings are 400/120(80)\n",
        "\n",
        "number_of_ticks_per_quarter = 420 #@param {type:\"slider\", min:10, max:500, step:10}\n",
        "encoding_has_MIDI_channels = False #@param {type:\"boolean\"}\n",
        "encoding_has_notes_velocities = False #@param {type:\"boolean\"}\n",
        "\n",
        "#fname = '/content/untitled'\n",
        "\n",
        "print('Converting TXT to MIDI. Please wait...')\n",
        "print('Converting TXT to Song...')\n",
        "output_list, song_name = TMIDI.Tegridy_Reduced_TXT_to_Notes_Converter(completion, has_MIDI_channels=encoding_has_MIDI_channels, has_velocities=encoding_has_notes_velocities, dataset_MIDI_events_time_denominator=10)\n",
        "\n",
        "print('Converting Song to MIDI...')\n",
        "\n",
        "output_signature = 'Efficient VIRTUOSO'\n",
        "\n",
        "detailed_stats = TMIDI.Tegridy_SONG_to_MIDI_Converter(output_list,\n",
        "                                                      output_signature = output_signature,  \n",
        "                                                      output_file_name = fname, \n",
        "                                                      track_name=song_name, \n",
        "                                                      number_of_ticks_per_quarter=number_of_ticks_per_quarter)\n",
        "\n",
        "print('Done!')\n",
        "\n",
        "print('Downloading your composition now...')\n",
        "from google.colab import files\n",
        "files.download(fname + '.mid')\n",
        "\n",
        "print('Detailed MIDI stats:')\n",
        "detailed_stats"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kILhoHR7JmmS",
        "cellView": "form"
      },
      "source": [
        "#@title Listen to the last generated composition\n",
        "#@markdown NOTE: May be very slow with the long compositions\n",
        "print('Synthesizing the last output MIDI. Please stand-by... ')\n",
        "FluidSynth(\"/usr/share/sounds/sf2/FluidR3_GM.sf2\", 16000).midi_to_audio(str(fname + '.mid'), str(fname + '.wav'))\n",
        "Audio(str(fname + '.wav'), rate=16000)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2Snu3fb4N-Nd"
      },
      "source": [
        "## Congrats! :) You did it :)"
      ]
    }
  ]
}