Update conversational widget to use text-generation (+ remove conversational task)

packages/inference/src/tasks/index.ts

@@ -18,7 +18,6 @@ export * from "./cv/imageToImage";
 export * from "./cv/zeroShotImageClassification";
 
 // Natural Language Processing tasks
-export * from "./nlp/conversational";
 export * from "./nlp/featureExtraction";
 export * from "./nlp/fillMask";
 export * from "./nlp/questionAnswering";

packages/inference/test/HfInference.spec.ts

@@ -333,25 +333,6 @@ describe.concurrent(
 				])
 			);
 		});
-		it("conversational", async () => {
-			expect(
-				await hf.conversational({
-					model: "microsoft/DialoGPT-large",
-					inputs: {
-						past_user_inputs: ["Which movie is the best ?"],
-						generated_responses: ["It is Die Hard for sure."],
-						text: "Can you explain why ?",
-					},
-				})
-			).toMatchObject({
-				generated_text: "It's the best movie ever.",
-				conversation: {
-					past_user_inputs: ["Which movie is the best ?", "Can you explain why ?"],
-					generated_responses: ["It is Die Hard for sure.", "It's the best movie ever."],
-				},
-				warnings: ["Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation."],
-			});
-		});
 		it("SentenceSimilarity", async () => {
 			expect(
 				await hf.sentenceSimilarity({

packages/tasks/src/default-widget-inputs.ts

@@ -1,9 +1,9 @@
 import type { WidgetExample } from "./widget-example";
-import type { PipelineType } from "./pipelines";
+import type { WidgetType } from "./pipelines";
 
 type LanguageCode = string;
 
-type PerLanguageMapping = Map<PipelineType, string[] | WidgetExample[]>;
+type PerLanguageMapping = Map<WidgetType, string[] | WidgetExample[]>;
 
 /// NOTE TO CONTRIBUTORS:
 ///

packages/tasks/src/index.ts

@@ -5,6 +5,7 @@ export * from "./tasks";
 export {
 	PIPELINE_DATA,
 	PIPELINE_TYPES,
+	type WidgetType,
 	type PipelineType,
 	type PipelineData,
 	type Modality,
@@ -16,6 +17,7 @@ export {
 export { ALL_DISPLAY_MODEL_LIBRARY_KEYS, ALL_MODEL_LIBRARY_KEYS, MODEL_LIBRARIES_UI_ELEMENTS } from "./model-libraries";
 export type { LibraryUiElement, ModelLibraryKey } from "./model-libraries";
 export type { ModelData, TransformersInfo } from "./model-data";
+export type { SpecialTokensMap, TokenizerConfig } from "./tokenizer-data";
 export type {
 	WidgetExample,
 	WidgetExampleAttribute,
@@ -37,6 +39,7 @@ export type {
 	WidgetExampleOutputText,
 } from "./widget-example";
 export { InferenceDisplayability } from "./model-data";
+export { SPECIAL_TOKENS_ATTRIBUTES } from "./tokenizer-data";
 
 import * as snippets from "./snippets";
 export { snippets };

packages/tasks/src/library-to-tasks.ts

@@ -27,7 +27,7 @@ export const LIBRARY_TASK_MAPPING_EXCLUDING_TRANSFORMERS: Partial<Record<ModelLi
 	keras: ["image-classification"],
 	nemo: ["automatic-speech-recognition"],
 	open_clip: ["zero-shot-classification", "zero-shot-image-classification"],
-	paddlenlp: ["conversational", "fill-mask", "summarization", "zero-shot-classification"],
+	paddlenlp: ["fill-mask", "summarization", "zero-shot-classification"],
 	peft: ["text-generation"],
 	"pyannote-audio": ["automatic-speech-recognition"],
 	"sentence-transformers": ["feature-extraction", "sentence-similarity"],

packages/tasks/src/model-data.ts

@@ -1,5 +1,6 @@
 import type { PipelineType } from "./pipelines";
 import type { WidgetExample } from "./widget-example";
+import type { TokenizerConfig } from "./tokenizer-data";
 
 export enum InferenceDisplayability {
 	/**
@@ -53,6 +54,7 @@ export interface ModelData {
 			base_model_name?: string;
 			task_type?: string;
 		};
+		tokenizer?: TokenizerConfig;
 	};
 	/**
 	 * all the model tags

packages/tasks/src/pipelines.ts

@@ -225,17 +225,6 @@ export const PIPELINE_DATA = {
 		modality: "nlp",
 		color: "indigo",
 	},
-	conversational: {
-		name: "Conversational",
-		subtasks: [
-			{
-				type: "dialogue-generation",
-				name: "Dialogue Generation",
-			},
-		],
-		modality: "nlp",
-		color: "green",
-	},
 	"feature-extraction": {
 		name: "Feature Extraction",
 		modality: "nlp",
@@ -248,6 +237,14 @@ export const PIPELINE_DATA = {
 				type: "dialogue-modeling",
 				name: "Dialogue Modeling",
 			},
+			{
+				type: "dialogue-generation",
+				name: "Dialogue Generation",
+			},
+			{
+				type: "conversational",
+				name: "Conversational",
+			},
 			{
 				type: "language-modeling",
 				name: "Language Modeling",
@@ -667,6 +664,8 @@ export const PIPELINE_DATA = {
 
 export type PipelineType = keyof typeof PIPELINE_DATA;
 
+export type WidgetType = PipelineType | "conversational";
+
 export const PIPELINE_TYPES = Object.keys(PIPELINE_DATA) as PipelineType[];
 
 export const SUBTASK_TYPES = Object.values(PIPELINE_DATA)

packages/tasks/src/snippets/curl.ts

@@ -34,7 +34,6 @@ export const curlSnippets: Partial<Record<PipelineType, (model: ModelData, acces
 	"zero-shot-classification": snippetZeroShotClassification,
 	translation: snippetBasic,
 	summarization: snippetBasic,
-	conversational: snippetBasic,
 	"feature-extraction": snippetBasic,
 	"text-generation": snippetBasic,
 	"text2text-generation": snippetBasic,

packages/tasks/src/snippets/inputs.ts

@@ -9,13 +9,6 @@ const inputsTranslation = () => `"Меня зовут Вольфганг и я 
 const inputsSummarization = () =>
 	`"The tower is 324 metres (1,063 ft) tall, about the same height as an 81-storey building, and the tallest structure in Paris. Its base is square, measuring 125 metres (410 ft) on each side. During its construction, the Eiffel Tower surpassed the Washington Monument to become the tallest man-made structure in the world, a title it held for 41 years until the Chrysler Building in New York City was finished in 1930. It was the first structure to reach a height of 300 metres. Due to the addition of a broadcasting aerial at the top of the tower in 1957, it is now taller than the Chrysler Building by 5.2 metres (17 ft). Excluding transmitters, the Eiffel Tower is the second tallest free-standing structure in France after the Millau Viaduct."`;
 
-const inputsConversational = () =>
-	`{
-		"past_user_inputs": ["Which movie is the best ?"],
-		"generated_responses": ["It is Die Hard for sure."],
-		"text": "Can you explain why ?"
-	}`;
-
 const inputsTableQuestionAnswering = () =>
 	`{
 		"query": "How many stars does the transformers repository have?",
@@ -96,7 +89,6 @@ const modelInputSnippets: {
 	"audio-to-audio": inputsAudioToAudio,
 	"audio-classification": inputsAudioClassification,
 	"automatic-speech-recognition": inputsAutomaticSpeechRecognition,
-	conversational: inputsConversational,
 	"document-question-answering": inputsVisualQuestionAnswering,
 	"feature-extraction": inputsFeatureExtraction,
 	"fill-mask": inputsFillMask,

packages/tasks/src/snippets/js.ts

@@ -121,7 +121,6 @@ export const jsSnippets: Partial<Record<PipelineType, (model: ModelData, accessT
 	"zero-shot-classification": snippetZeroShotClassification,
 	translation: snippetBasic,
 	summarization: snippetBasic,
-	conversational: snippetBasic,
 	"feature-extraction": snippetBasic,
 	"text-generation": snippetBasic,
 	"text2text-generation": snippetBasic,

packages/tasks/src/snippets/python.ts

@@ -116,7 +116,6 @@ export const pythonSnippets: Partial<Record<PipelineType, (model: ModelData) =>
 	"zero-shot-classification": snippetZeroShotClassification,
 	translation: snippetBasic,
 	summarization: snippetBasic,
-	conversational: snippetBasic,
 	"feature-extraction": snippetBasic,
 	"text-generation": snippetBasic,
 	"text2text-generation": snippetBasic,