From 678eba85a800f73d2c62408bfd30206504c8f0bd Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Mon, 2 Sep 2024 14:28:32 +0300 Subject: [PATCH 01/10] Add keypoint-detection task to Hub (#870) --------- Co-authored-by: Merve Noyan Co-authored-by: Pedro Cuenca --- packages/tasks/src/pipelines.ts | 12 ++++ packages/tasks/src/tasks/index.ts | 2 + .../src/tasks/keypoint-detection/about.md | 59 +++++++++++++++++++ .../src/tasks/keypoint-detection/data.ts | 46 +++++++++++++++ .../Icons/IconKeypointDetection.svelte | 1 + .../PipelineIcon/PipelineIcon.svelte | 2 + 6 files changed, 122 insertions(+) create mode 100644 packages/tasks/src/tasks/keypoint-detection/about.md create mode 100644 packages/tasks/src/tasks/keypoint-detection/data.ts create mode 100644 packages/widgets/src/lib/components/Icons/IconKeypointDetection.svelte diff --git a/packages/tasks/src/pipelines.ts b/packages/tasks/src/pipelines.ts index 3c9d9a6ac..7edc61605 100644 --- a/packages/tasks/src/pipelines.ts +++ b/packages/tasks/src/pipelines.ts @@ -656,6 +656,18 @@ export const PIPELINE_DATA = { name: "Video-Text-to-Text", modality: "multimodal", color: "blue", + hideInDatasets: false, + }, + "keypoint-detection": { + name: "Keypoint Detection", + subtasks: [ + { + type: "pose-estimation", + name: "Pose Estimation", + }, + ], + modality: "cv", + color: "red", hideInDatasets: true, }, other: { diff --git a/packages/tasks/src/tasks/index.ts b/packages/tasks/src/tasks/index.ts index b3c0e0e8e..a72bb9c88 100644 --- a/packages/tasks/src/tasks/index.ts +++ b/packages/tasks/src/tasks/index.ts @@ -126,6 +126,7 @@ export const TASKS_MODEL_LIBRARIES: Record = { "image-to-image": ["diffusers", "transformers", "transformers.js"], "image-to-text": ["transformers", "transformers.js"], "image-to-video": ["diffusers"], + "keypoint-detection": ["transformers"], "video-classification": ["transformers"], "mask-generation": ["transformers"], "multiple-choice": ["transformers"], @@ -205,6 +206,7 @@ export const TASKS_DATA: Record = { "image-text-to-text": getData("image-text-to-text", imageTextToText), "image-to-text": getData("image-to-text", imageToText), "image-to-video": undefined, + "keypoint-detection": getData("keypoint-detection", placeholder), "mask-generation": getData("mask-generation", maskGeneration), "multiple-choice": undefined, "object-detection": getData("object-detection", objectDetection), diff --git a/packages/tasks/src/tasks/keypoint-detection/about.md b/packages/tasks/src/tasks/keypoint-detection/about.md new file mode 100644 index 000000000..7067695cc --- /dev/null +++ b/packages/tasks/src/tasks/keypoint-detection/about.md @@ -0,0 +1,59 @@ +## Task Variants + +### Pose Estimation + +Pose estimation is the process of determining the position and orientation of an object or a camera in a 3D space. It is a fundamental task in computer vision and is widely used in various applications such as robotics, augmented reality, and 3D reconstruction. + +## Use Cases for Keypoint Detection + +### Facial Landmark Estimation + +Keypoint detection models can be used to estimate the position of facial landmarks. Facial landmarks are points on the face such as the corners of the mouth, the outer corners of the eyes, and the tip of the nose. These landmarks can be used for a variety of applications, such as facial expression recognition, 3D face reconstruction, and cinematic animation. + +### Fitness Tracking + +Keypoint detection models can be used to track the movement of the human body, e.g. position of the joints in a 3D space. This can be used for a variety of applications, such as fitness tracking, sports analysis or virtual reality applications. + +## Inference Code + +Below you can find an example of how to use a keypoint detection model and how to visualize the results. + +```python +from transformers import AutoImageProcessor, SuperPointForKeypointDetection +import torch +import matplotlib.pyplot as plt +from PIL import Image +import requests + +url_image = "http://images.cocodataset.org/val2017/000000039769.jpg" +image = Image.open(requests.get(url_image_1, stream=True).raw) + +# initialize the model and processor +processor = AutoImageProcessor.from_pretrained("magic-leap-community/superpoint") +model = SuperPointForKeypointDetection.from_pretrained("magic-leap-community/superpoint") + +# infer +inputs = processor(image, return_tensors="pt").to(model.device, model.dtype) +outputs = model(**inputs) + +# visualize the output +image_width, image_height = image.size +image_mask = outputs.mask +image_indices = torch.nonzero(image_mask).squeeze() + +image_scores = outputs.scores.squeeze() +image_keypoints = outputs.keypoints.squeeze() +keypoints = image_keypoints.detach().numpy() +scores = image_scores.detach().numpy() + +plt.axis('off') +plt.imshow(image) +plt.scatter( + keypoints[:, 0], + keypoints[:, 1], + s=scores * 100, + c='cyan', + alpha=0.4 +) +plt.show() +``` diff --git a/packages/tasks/src/tasks/keypoint-detection/data.ts b/packages/tasks/src/tasks/keypoint-detection/data.ts new file mode 100644 index 000000000..6b029d0e0 --- /dev/null +++ b/packages/tasks/src/tasks/keypoint-detection/data.ts @@ -0,0 +1,46 @@ +import type { TaskDataCustom } from ".."; + +const taskData: TaskDataCustom = { + datasets: [ + { + description: "A dataset of hand keypoints of over 500k examples.", + id: "Vincent-luo/hagrid-mediapipe-hands", + }, + ], + demo: { + inputs: [ + { + filename: "keypoint-detection-input.png", + type: "img", + }, + ], + outputs: [ + { + filename: "keypoint-detection-output.png", + type: "img", + }, + ], + }, + metrics: [], + models: [ + { + description: "A robust keypoint detection model.", + id: "magic-leap-community/superpoint", + }, + { + description: "Strong keypoint detection model used to detect human pose.", + id: "qualcomm/MediaPipe-Pose-Estimation", + }, + ], + spaces: [ + { + description: "An application that detects hand keypoints in real-time.", + id: "datasciencedojo/Hand-Keypoint-Detection-Realtime", + }, + ], + summary: "Keypoint detection is the task of identifying meaningful distinctive points or features in an image.", + widgetModels: [], + youtubeId: "", +}; + +export default taskData; diff --git a/packages/widgets/src/lib/components/Icons/IconKeypointDetection.svelte b/packages/widgets/src/lib/components/Icons/IconKeypointDetection.svelte new file mode 100644 index 000000000..312d1b11c --- /dev/null +++ b/packages/widgets/src/lib/components/Icons/IconKeypointDetection.svelte @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte index cd15d43f3..02cb84f27 100644 --- a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte +++ b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte @@ -43,6 +43,7 @@ import IconImageTo3D from "../Icons/IconImageTo3D.svelte"; import IconImageFeatureExtraction from "../Icons/IconImageFeatureExtraction.svelte"; import IconVideoTextToText from "../Icons/IconVideoTextToText.svelte"; + import IconKeypointDetection from "../Icons/IconKeypointDetection.svelte"; import type { WidgetType } from "@huggingface/tasks"; export let classNames = ""; @@ -96,6 +97,7 @@ "image-to-3d": IconImageTo3D, "image-feature-extraction": IconImageFeatureExtraction, "video-text-to-text": IconVideoTextToText, + "keypoint-detection": IconKeypointDetection, }; $: iconComponent = From 3bc437cafd121ce7d2b72ff01ec24885a214ba5f Mon Sep 17 00:00:00 2001 From: pngwn Date: Mon, 2 Sep 2024 22:06:20 +0800 Subject: [PATCH 02/10] Handle user and org avatars (#816) Currently if there is no avatar then the space header can display a broken image link. This PR adds a simple check to see if the avatar exists. If it doesn't then we don't add the DOM for it. I decided to do it outside of the 'component' rendering just to keep things cleaner + sync after the main function but I'm happy to change this. Could be optimised to make the fetches in parallel but the difference is probably minor in most cases. Screenshot: Screenshot 2024-07-23 at 10 19 56 cc @enzostvs @coyotte508 @julien-c --------- Co-authored-by: enzo --- .../src/header/components/content/avatar.ts | 6 ++++-- .../src/header/components/content/index.ts | 4 +++- packages/space-header/src/index.ts | 6 +++++- packages/space-header/src/type.ts | 1 + packages/space-header/src/utils/check_avatar.ts | 10 ++++++++++ packages/space-header/src/{ => utils}/get_space.ts | 3 ++- 6 files changed, 25 insertions(+), 5 deletions(-) create mode 100644 packages/space-header/src/utils/check_avatar.ts rename packages/space-header/src/{ => utils}/get_space.ts (81%) diff --git a/packages/space-header/src/header/components/content/avatar.ts b/packages/space-header/src/header/components/content/avatar.ts index 88445faa1..468b6b5a9 100644 --- a/packages/space-header/src/header/components/content/avatar.ts +++ b/packages/space-header/src/header/components/content/avatar.ts @@ -1,6 +1,8 @@ -export const Avatar = (username: string): HTMLImageElement => { +export const Avatar = (username: string, type: "user" | "org" = "user"): HTMLImageElement => { + const route = type === "user" ? "users" : "organizations"; + const element = document.createElement("img"); - element.src = `https://huggingface.co/api/users/${username}/avatar`; + element.src = `https://huggingface.co/api/${route}/${username}/avatar`; element.style.width = "0.875rem"; element.style.height = "0.875rem"; diff --git a/packages/space-header/src/header/components/content/index.ts b/packages/space-header/src/header/components/content/index.ts index b1679a8b0..a1b86705c 100644 --- a/packages/space-header/src/header/components/content/index.ts +++ b/packages/space-header/src/header/components/content/index.ts @@ -15,7 +15,9 @@ export const Content = (space: Space): HTMLDivElement => { content.style.paddingRight = "12px"; content.style.height = "40px"; - content.appendChild(Avatar(space.author)); + if (space.type !== "unknown") { + content.appendChild(Avatar(space.author, space.type)); + } content.appendChild(Username(space.author)); content.appendChild(Separation()); content.appendChild(Namespace(space.id)); diff --git a/packages/space-header/src/index.ts b/packages/space-header/src/index.ts index 6c66a5fba..35fe129d1 100644 --- a/packages/space-header/src/index.ts +++ b/packages/space-header/src/index.ts @@ -3,7 +3,8 @@ import type { Options, Space, Header } from "./type"; import { inject_fonts } from "./inject_fonts"; import { create } from "./header/create"; -import { get_space } from "./get_space"; +import { check_avatar } from "./utils/check_avatar"; +import { get_space } from "./utils/get_space"; import { inject } from "./inject"; async function main(initialSpace: string | Space, options?: Options) { @@ -27,6 +28,9 @@ async function main(initialSpace: string | Space, options?: Options) { space = initialSpace; } + const [user, org] = await Promise.all([check_avatar(space.author, "user"), check_avatar(space.author, "org")]); + space.type = user ? "user" : org ? "org" : "unknown"; + const mini_header_element = create(space as Space); inject(mini_header_element, options); diff --git a/packages/space-header/src/type.ts b/packages/space-header/src/type.ts index ac50d3420..c4fe97fa5 100644 --- a/packages/space-header/src/type.ts +++ b/packages/space-header/src/type.ts @@ -2,6 +2,7 @@ export interface Space { id: string; likes: number; author: string; + type?: "user" | "org" | "unknown"; } export interface User { diff --git a/packages/space-header/src/utils/check_avatar.ts b/packages/space-header/src/utils/check_avatar.ts new file mode 100644 index 000000000..0faa666a0 --- /dev/null +++ b/packages/space-header/src/utils/check_avatar.ts @@ -0,0 +1,10 @@ +export const check_avatar = async (username: string, type: "user" | "org" = "user"): Promise => { + const route = type === "user" ? "users" : "organizations"; + + try { + const response = await fetch(`https://huggingface.co/api/${route}/${username}/avatar`); + return response.ok; + } catch (error) { + return false; + } +}; diff --git a/packages/space-header/src/get_space.ts b/packages/space-header/src/utils/get_space.ts similarity index 81% rename from packages/space-header/src/get_space.ts rename to packages/space-header/src/utils/get_space.ts index 84a11ddeb..608350bd1 100644 --- a/packages/space-header/src/get_space.ts +++ b/packages/space-header/src/utils/get_space.ts @@ -1,9 +1,10 @@ -import type { Space } from "./type"; +import type { Space } from "./../type"; export const get_space = async (space_id: string): Promise => { try { const response = await fetch(`https://huggingface.co/api/spaces/${space_id}`); const data = await response.json(); + console.log(data); return data as Space; } catch (error) { return null; From fa428c2e2b0a7eb6dd8e5f0bc76e913fd311320f Mon Sep 17 00:00:00 2001 From: Luke Chang Date: Mon, 2 Sep 2024 11:35:00 -0400 Subject: [PATCH 03/10] register py-feat library (#879) The goal of this PR is to register a new library [py-feat](https://py-feat.org/) with huggingface and add supported tasks of `image-feature-extraction`. Please let me know if I should make any changes. --------- Co-authored-by: Lucain --- packages/tasks/src/model-libraries.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index 71b8747b9..55bf8d09c 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -446,6 +446,13 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { snippets: snippets.pyannote_audio, filter: true, }, + "py-feat": { + prettyLabel: "Py-Feat", + repoName: "Py-Feat", + repoUrl: "https://github.com/cosanlab/py-feat", + docsUrl: "https://py-feat.org/", + filter: false, + }, pythae: { prettyLabel: "pythae", repoName: "pythae", From 82d822473fdcec1dd206ff1ac1576515046ce763 Mon Sep 17 00:00:00 2001 From: machineuser Date: Mon, 2 Sep 2024 16:55:39 +0000 Subject: [PATCH 04/10] =?UTF-8?q?=F0=9F=94=96=20@hugginface/tasks=200.11.1?= =?UTF-8?q?3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/tasks/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/package.json b/packages/tasks/package.json index 06ea1cc0b..25c37c4ae 100644 --- a/packages/tasks/package.json +++ b/packages/tasks/package.json @@ -1,7 +1,7 @@ { "name": "@huggingface/tasks", "packageManager": "pnpm@8.10.5", - "version": "0.11.12", + "version": "0.11.13", "description": "List of ML tasks for huggingface.co/tasks", "repository": "https://github.com/huggingface/huggingface.js.git", "publishConfig": { From 1701fac9088845de393a924f12bfd09f9ca70056 Mon Sep 17 00:00:00 2001 From: Lucain Date: Mon, 2 Sep 2024 18:59:59 +0200 Subject: [PATCH 05/10] Add seed in text to image specs (#888) Following @apolinario's PR https://github.com/huggingface/api-inference-community/pull/450. This PR adds a "seed" input parameter in the `text-to-image` specs. --------- Co-authored-by: Pedro Cuenca --- .../tasks/src/tasks/text-to-image/inference.ts | 14 +++++++++----- .../tasks/src/tasks/text-to-image/spec/input.json | 10 +++++++--- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/packages/tasks/src/tasks/text-to-image/inference.ts b/packages/tasks/src/tasks/text-to-image/inference.ts index b2e735746..8c30d3e9e 100644 --- a/packages/tasks/src/tasks/text-to-image/inference.ts +++ b/packages/tasks/src/tasks/text-to-image/inference.ts @@ -26,8 +26,8 @@ export interface TextToImageInput { */ export interface TextToImageParameters { /** - * For diffusion models. A higher guidance scale value encourages the model to generate - * images closely linked to the text prompt at the expense of lower image quality. + * A higher guidance scale value encourages the model to generate images closely linked to + * the text prompt, but values too high may cause saturation and other artifacts. */ guidance_scale?: number; /** @@ -35,14 +35,18 @@ export interface TextToImageParameters { */ negative_prompt?: string[]; /** - * For diffusion models. The number of denoising steps. More denoising steps usually lead to - * a higher quality image at the expense of slower inference. + * The number of denoising steps. More denoising steps usually lead to a higher quality + * image at the expense of slower inference. */ num_inference_steps?: number; /** - * For diffusion models. Override the scheduler with a compatible one + * Override the scheduler with a compatible one. */ scheduler?: string; + /** + * Seed for the random number generator. + */ + seed?: number; /** * The size in pixel of the output image */ diff --git a/packages/tasks/src/tasks/text-to-image/spec/input.json b/packages/tasks/src/tasks/text-to-image/spec/input.json index 467b848f6..569f3c33a 100644 --- a/packages/tasks/src/tasks/text-to-image/spec/input.json +++ b/packages/tasks/src/tasks/text-to-image/spec/input.json @@ -22,7 +22,7 @@ "properties": { "guidance_scale": { "type": "number", - "description": "For diffusion models. A higher guidance scale value encourages the model to generate images closely linked to the text prompt at the expense of lower image quality." + "description": "A higher guidance scale value encourages the model to generate images closely linked to the text prompt, but values too high may cause saturation and other artifacts." }, "negative_prompt": { "type": "array", @@ -33,7 +33,7 @@ }, "num_inference_steps": { "type": "integer", - "description": "For diffusion models. The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference." + "description": "The number of denoising steps. More denoising steps usually lead to a higher quality image at the expense of slower inference." }, "target_size": { "type": "object", @@ -50,7 +50,11 @@ }, "scheduler": { "type": "string", - "description": "For diffusion models. Override the scheduler with a compatible one" + "description": "Override the scheduler with a compatible one." + }, + "seed": { + "type": "integer", + "description": "Seed for the random number generator." } } } From e0d6c958dc3a5ce3c67b46d10d7f5224557a346d Mon Sep 17 00:00:00 2001 From: Linoy Tsaban <57615435+linoytsaban@users.noreply.github.com> Date: Tue, 3 Sep 2024 09:54:09 +0300 Subject: [PATCH 06/10] improve image-to-image task page (#867) some changes to improve clarity of task description, and general updates to improve task page --------- Co-authored-by: Pedro Cuenca Co-authored-by: Merve Noyan Co-authored-by: Omar Sanseviero --- .../tasks/src/tasks/image-to-image/about.md | 91 ++++++++++++++----- .../tasks/src/tasks/image-to-image/data.ts | 2 +- 2 files changed, 71 insertions(+), 22 deletions(-) diff --git a/packages/tasks/src/tasks/image-to-image/about.md b/packages/tasks/src/tasks/image-to-image/about.md index 63f490f82..3750b34e5 100644 --- a/packages/tasks/src/tasks/image-to-image/about.md +++ b/packages/tasks/src/tasks/image-to-image/about.md @@ -1,15 +1,10 @@ -## Use Cases - -### Style transfer +Image-to-image pipelines can also be used in text-to-image tasks, to provide visual guidance to the text-guided generation process. -One of the most popular use cases of image-to-image is style transfer. Style transfer models can convert a normal photography into a painting in the style of a famous painter. - -## Task Variants +## Use Cases ### Image inpainting -Image inpainting is widely used during photography editing to remove unwanted objects, such as poles, wires, or sensor -dust. +Image inpainting is widely used during photography editing to remove unwanted objects, such as poles, wires, or sensor dust. ### Image colorization @@ -24,18 +19,27 @@ Super-resolution models increase the resolution of an image, allowing for higher You can use pipelines for image-to-image in ๐Ÿงจdiffusers library to easily use image-to-image models. See an example for `StableDiffusionImg2ImgPipeline` below. ```python -from PIL import Image -from diffusers import StableDiffusionImg2ImgPipeline +import torch +from diffusers import AutoPipelineForImage2Image +from diffusers.utils import make_image_grid, load_image -model_id_or_path = "runwayml/stable-diffusion-v1-5" -pipe = StableDiffusionImg2ImgPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16) -pipe = pipe.to(cuda) +pipeline = AutoPipelineForImage2Image.from_pretrained( + "stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16, variant="fp16", use_safetensors=True +) -init_image = Image.open("mountains_image.jpeg").convert("RGB").resize((768, 512)) -prompt = "A fantasy landscape, trending on artstation" +# this helps us to reduce memory usage- since SDXL is a bit heavy, this could help by +# offloading the model to CPU w/o hurting performance. +pipeline.enable_model_cpu_offload() -images = pipe(prompt=prompt, image=init_image, strength=0.75, guidance_scale=7.5).images -images[0].save("fantasy_landscape.png") +# prepare image +url = "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/img2img-sdxl-init.png" +init_image = load_image(url) + +prompt = "Astronaut in a jungle, cold color palette, muted colors, detailed, 8k" + +# pass prompt and image to pipeline +image = pipeline(prompt, image=init_image, strength=0.5).images[0] +make_image_grid([init_image, image], rows=1, cols=2) ``` You can use [huggingface.js](https://github.com/huggingface/huggingface.js) to infer image-to-image models on Hugging Face Hub. @@ -53,13 +57,53 @@ await inference.imageToImage({ }); ``` -## ControlNet +## Uses Cases for Text Guided Image Generation -Controlling the outputs of diffusion models only with a text prompt is a challenging problem. ControlNet is a neural network model that provides image-based control to diffusion models. Control images can be edges or other landmarks extracted from a source image. +### Style Transfer + +One of the most popular use cases of image-to-image is style transfer. With style transfer models: -Many ControlNet models were trained in our community event, JAX Diffusers sprint. You can see the full list of the ControlNet models available [here](https://huggingface.co/spaces/jax-diffusers-event/leaderboard). +- a regular photo can be transformed into a variety of artistic styles or genres, such as a watercolor painting, a comic book illustration and more. +- new images can be generated using a text prompt, in the style of a reference input image. + +See ๐Ÿงจdiffusers example for style transfer with `AutoPipelineForText2Image` below. + +```python +from diffusers import AutoPipelineForText2Image +from diffusers.utils import load_image +import torch + +# load pipeline +pipeline = AutoPipelineForText2Image.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16).to("cuda") +pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin") + +# set the adapter and scales - this is a component that lets us add the style control from an image to the text-to-image model +scale = { + "down": {"block_2": [0.0, 1.0]}, + "up": {"block_0": [0.0, 1.0, 0.0]}, +} +pipeline.set_ip_adapter_scale(scale) + +style_image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/0052a70beed5bf71b92610a43a52df6d286cd5f3/diffusers/rabbit.jpg") + +generator = torch.Generator(device="cpu").manual_seed(26) +image = pipeline( + prompt="a cat, masterpiece, best quality, high quality", + ip_adapter_image=style_image, + negative_prompt="text, watermark, lowres, low quality, worst quality, deformed, glitch, low contrast, noisy, saturation, blurry", + guidance_scale=5, + num_inference_steps=30, + generator=generator, +).images[0] +image +``` + +### ControlNet + +Controlling the outputs of diffusion models only with a text prompt is a challenging problem. ControlNet is a neural network model that provides image-based control to diffusion models. Control images can be edges or other landmarks extracted from a source image. +![Examples](https://huggingface.co/datasets/optimum/documentation-images/resolve/main/neuron/models/12-sdxl-text2img-controlnet.png) -## Most Used Model for the Task +## Pix2Pix Pix2Pix is a popular model used for image-to-image translation tasks. It is based on a conditional-GAN (generative adversarial network) where instead of a noise vector a 2D image is given as input. More information about Pix2Pix can be retrieved from this [link](https://phillipi.github.io/pix2pix/) where the associated paper and the GitHub repository can be found. @@ -70,8 +114,13 @@ The images below show some examples extracted from the Pix2Pix paper. This model ## Useful Resources - [Image-to-image guide with diffusers](https://huggingface.co/docs/diffusers/using-diffusers/img2img) +- Image inpainting: [inpainting with ๐Ÿงจdiffusers](https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/inpaint), [demo](https://huggingface.co/spaces/diffusers/stable-diffusion-xl-inpainting) +- Colorization: [demo](https://huggingface.co/spaces/modelscope/old_photo_restoration) +- Super resolution: [image upscaling with ๐Ÿงจdiffusers](https://huggingface.co/docs/diffusers/main/en/api/pipelines/stable_diffusion/upscale#super-resolution), [demo](https://huggingface.co/spaces/radames/Enhance-This-HiDiffusion-SDXL) +- [Style transfer and layout control with diffusers ๐Ÿงจ](https://huggingface.co/docs/diffusers/main/en/using-diffusers/ip_adapter#style--layout-control) - [Train your ControlNet with diffusers ๐Ÿงจ](https://huggingface.co/blog/train-your-controlnet) - [Ultra fast ControlNet with ๐Ÿงจ Diffusers](https://huggingface.co/blog/controlnet) +- [List of ControlNets trained in the community JAX Diffusers sprint](https://huggingface.co/spaces/jax-diffusers-event/leaderboard) ## References diff --git a/packages/tasks/src/tasks/image-to-image/data.ts b/packages/tasks/src/tasks/image-to-image/data.ts index 99e91557a..65200fd92 100644 --- a/packages/tasks/src/tasks/image-to-image/data.ts +++ b/packages/tasks/src/tasks/image-to-image/data.ts @@ -93,7 +93,7 @@ const taskData: TaskDataCustom = { }, ], summary: - "Image-to-image is the task of transforming a source image to match the characteristics of a target image or a target image domain. Any image manipulation and enhancement is possible with image to image models.", + "Image-to-image is the task of transforming an input image through a variety of possible manipulations and enhancements, such as super-resolution, image inpainting, colorization, and more.", widgetModels: ["lllyasviel/sd-controlnet-canny"], youtubeId: "", }; From 1c9a2a281065a6da581317caeb4d5bd4174b7645 Mon Sep 17 00:00:00 2001 From: enzo Date: Tue, 3 Sep 2024 13:20:15 -0400 Subject: [PATCH 07/10] remove unused console.log (#891) --- packages/space-header/src/utils/get_space.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/space-header/src/utils/get_space.ts b/packages/space-header/src/utils/get_space.ts index 608350bd1..4250b2206 100644 --- a/packages/space-header/src/utils/get_space.ts +++ b/packages/space-header/src/utils/get_space.ts @@ -4,7 +4,6 @@ export const get_space = async (space_id: string): Promise => { try { const response = await fetch(`https://huggingface.co/api/spaces/${space_id}`); const data = await response.json(); - console.log(data); return data as Space; } catch (error) { return null; From 787c7cae842361a55d3808f6cacb4c2ba6bd2234 Mon Sep 17 00:00:00 2001 From: machineuser Date: Tue, 3 Sep 2024 17:24:52 +0000 Subject: [PATCH 08/10] =?UTF-8?q?=F0=9F=94=96=20@hugginface/space-header?= =?UTF-8?q?=201.0.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/space-header/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/space-header/package.json b/packages/space-header/package.json index 735315c8d..c9c2c43c9 100644 --- a/packages/space-header/package.json +++ b/packages/space-header/package.json @@ -1,6 +1,6 @@ { "name": "@huggingface/space-header", - "version": "1.0.3", + "version": "1.0.4", "packageManager": "pnpm@8.10.5", "description": "Use the Space mini_header outside Hugging Face", "repository": "https://github.com/huggingface/huggingface.js.git", From a9047d5af078a6bc8b649bde8677195c528d9f10 Mon Sep 17 00:00:00 2001 From: Lucain Date: Wed, 4 Sep 2024 09:15:30 +0200 Subject: [PATCH 09/10] We are Hugging Face, or huggingface, or Huggingface, who knows (#893) --- .github/workflows/agents-publish.yml | 2 +- .github/workflows/gguf-publish.yml | 2 +- .github/workflows/hub-publish.yml | 2 +- .github/workflows/inference-publish.yml | 2 +- .github/workflows/jinja-publish.yml | 2 +- .github/workflows/languages-publish.yml | 2 +- .github/workflows/space-header-publish.yml | 2 +- .github/workflows/tasks-publish.yml | 2 +- packages/doc-internal/README.md | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/agents-publish.yml b/.github/workflows/agents-publish.yml index 08108b585..edc72455f 100644 --- a/.github/workflows/agents-publish.yml +++ b/.github/workflows/agents-publish.yml @@ -50,7 +50,7 @@ jobs: node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" pnpm --filter doc-internal run fix-cdn-versions git add ../.. - git commit -m "๐Ÿ”– @hugginface/agents $BUMPED_VERSION" + git commit -m "๐Ÿ”– @huggingface/agents $BUMPED_VERSION" git tag "agents-v$BUMPED_VERSION" - run: pnpm --filter agents... build && pnpm publish --no-git-checks . env: diff --git a/.github/workflows/gguf-publish.yml b/.github/workflows/gguf-publish.yml index dec6cd891..5e5cf81f0 100644 --- a/.github/workflows/gguf-publish.yml +++ b/.github/workflows/gguf-publish.yml @@ -47,7 +47,7 @@ jobs: BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')") # Update package.json with the new version node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" - git commit . -m "๐Ÿ”– @hugginface/gguf $BUMPED_VERSION" + git commit . -m "๐Ÿ”– @huggingface/gguf $BUMPED_VERSION" git tag "gguf-v$BUMPED_VERSION" - run: pnpm publish --no-git-checks . env: diff --git a/.github/workflows/hub-publish.yml b/.github/workflows/hub-publish.yml index 19bb63dcf..5937ad212 100644 --- a/.github/workflows/hub-publish.yml +++ b/.github/workflows/hub-publish.yml @@ -50,7 +50,7 @@ jobs: node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" pnpm --filter doc-internal run fix-cdn-versions git add ../.. - git commit -m "๐Ÿ”– @hugginface/hub $BUMPED_VERSION" + git commit -m "๐Ÿ”– @huggingface/hub $BUMPED_VERSION" git tag "hub-v$BUMPED_VERSION" - name: Make sure that the latest version of @huggingface/tasks is consistent with the local version diff --git a/.github/workflows/inference-publish.yml b/.github/workflows/inference-publish.yml index 5ba387645..bdbb47d7d 100644 --- a/.github/workflows/inference-publish.yml +++ b/.github/workflows/inference-publish.yml @@ -50,7 +50,7 @@ jobs: node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" pnpm --filter doc-internal run fix-cdn-versions git add ../.. - git commit -m "๐Ÿ”– @hugginface/inference $BUMPED_VERSION" + git commit -m "๐Ÿ”– @huggingface/inference $BUMPED_VERSION" git tag "inference-v$BUMPED_VERSION" - name: Make sure that the latest version of @huggingface/tasks is consistent with the local version diff --git a/.github/workflows/jinja-publish.yml b/.github/workflows/jinja-publish.yml index 5057d4ed5..47d7cad7f 100644 --- a/.github/workflows/jinja-publish.yml +++ b/.github/workflows/jinja-publish.yml @@ -47,7 +47,7 @@ jobs: BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')") # Update package.json with the new version node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" - git commit . -m "๐Ÿ”– @hugginface/jinja $BUMPED_VERSION" + git commit . -m "๐Ÿ”– @huggingface/jinja $BUMPED_VERSION" git tag "jinja-v$BUMPED_VERSION" - run: pnpm publish --no-git-checks . env: diff --git a/.github/workflows/languages-publish.yml b/.github/workflows/languages-publish.yml index 5dca90f89..913687a52 100644 --- a/.github/workflows/languages-publish.yml +++ b/.github/workflows/languages-publish.yml @@ -47,7 +47,7 @@ jobs: BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')") # Update package.json with the new version node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" - git commit . -m "๐Ÿ”– @hugginface/languages $BUMPED_VERSION" + git commit . -m "๐Ÿ”– @huggingface/languages $BUMPED_VERSION" git tag "languages-v$BUMPED_VERSION" - run: pnpm publish --no-git-checks . env: diff --git a/.github/workflows/space-header-publish.yml b/.github/workflows/space-header-publish.yml index 76ce2d89b..05c584514 100644 --- a/.github/workflows/space-header-publish.yml +++ b/.github/workflows/space-header-publish.yml @@ -47,7 +47,7 @@ jobs: BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')") # Update package.json with the new version node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" - git commit . -m "๐Ÿ”– @hugginface/space-header $BUMPED_VERSION" + git commit . -m "๐Ÿ”– @huggingface/space-header $BUMPED_VERSION" git tag "space-header-v$BUMPED_VERSION" - run: pnpm publish --no-git-checks . env: diff --git a/.github/workflows/tasks-publish.yml b/.github/workflows/tasks-publish.yml index 4c8b4567e..0dd797dd2 100644 --- a/.github/workflows/tasks-publish.yml +++ b/.github/workflows/tasks-publish.yml @@ -47,7 +47,7 @@ jobs: BUMPED_VERSION=$(node -p "require('semver').inc('$PACKAGE_VERSION', '${{ github.event.inputs.newversion }}')") # Update package.json with the new version node -e "const fs = require('fs'); const package = JSON.parse(fs.readFileSync('./package.json')); package.version = '$BUMPED_VERSION'; fs.writeFileSync('./package.json', JSON.stringify(package, null, '\t') + '\n');" - git commit . -m "๐Ÿ”– @hugginface/tasks $BUMPED_VERSION" + git commit . -m "๐Ÿ”– @huggingface/tasks $BUMPED_VERSION" git tag "tasks-v$BUMPED_VERSION" - run: pnpm publish --no-git-checks . env: diff --git a/packages/doc-internal/README.md b/packages/doc-internal/README.md index 8d51e72ab..71d8f8314 100644 --- a/packages/doc-internal/README.md +++ b/packages/doc-internal/README.md @@ -2,7 +2,7 @@ This package generates `.md` files inside the [docs](../../docs) folder using [typedoc](https://typedoc.org/) and [typedoc-plugin-markdown](https://github.com/tgreyuk/typedoc-plugin-markdown). -The `.md` files are generated when releasing packages. They are then published to [hugginface.co](https://huggingface.co/docs/huggingface.js/index) through the [doc-builder](https://github.com/huggingface/doc-builder)'s github action. +The `.md` files are generated when releasing packages. They are then published to [huggingface.co](https://huggingface.co/docs/huggingface.js/index) through the [doc-builder](https://github.com/huggingface/doc-builder)'s github action. We run a few scripts in between, [fix-md-links](./fix-md-links.ts) and [update-toc](./update-toc.ts) to preprocess the files for `doc-builder`. From 6e12cfae0a4535467290b9b6b045282b5a91cd8a Mon Sep 17 00:00:00 2001 From: Merve Noyan Date: Wed, 4 Sep 2024 11:15:53 +0300 Subject: [PATCH 10/10] Add any-to-any as a task to Hub (#860) This PR adds any-to-any for tasks that have 2+ inputs to 2+ outputs like 4M, Chameleon, Lumina-mGPT etc --- packages/tasks/src/pipelines.ts | 6 ++++++ packages/tasks/src/tasks/index.ts | 2 ++ .../widgets/src/lib/components/Icons/IconAnyToAny.svelte | 1 + .../src/lib/components/PipelineIcon/PipelineIcon.svelte | 2 ++ 4 files changed, 11 insertions(+) create mode 100644 packages/widgets/src/lib/components/Icons/IconAnyToAny.svelte diff --git a/packages/tasks/src/pipelines.ts b/packages/tasks/src/pipelines.ts index 7edc61605..34907f064 100644 --- a/packages/tasks/src/pipelines.ts +++ b/packages/tasks/src/pipelines.ts @@ -670,6 +670,12 @@ export const PIPELINE_DATA = { color: "red", hideInDatasets: true, }, + "any-to-any": { + name: "Any-to-Any", + modality: "multimodal", + color: "yellow", + hideInDatasets: true, + }, other: { name: "Other", modality: "other", diff --git a/packages/tasks/src/tasks/index.ts b/packages/tasks/src/tasks/index.ts index a72bb9c88..6c8068680 100644 --- a/packages/tasks/src/tasks/index.ts +++ b/packages/tasks/src/tasks/index.ts @@ -170,6 +170,7 @@ export const TASKS_MODEL_LIBRARIES: Record = { "zero-shot-object-detection": ["transformers", "transformers.js"], "text-to-3d": ["diffusers"], "image-to-3d": ["diffusers"], + "any-to-any": ["transformers"], }; /** @@ -191,6 +192,7 @@ function getData(type: PipelineType, partialTaskData: TaskDataCustom = placehold // Tasks that call getData() without the second argument will // have a "placeholder" page. export const TASKS_DATA: Record = { + "any-to-any": getData("any-to-any", placeholder), "audio-classification": getData("audio-classification", audioClassification), "audio-to-audio": getData("audio-to-audio", audioToAudio), "automatic-speech-recognition": getData("automatic-speech-recognition", automaticSpeechRecognition), diff --git a/packages/widgets/src/lib/components/Icons/IconAnyToAny.svelte b/packages/widgets/src/lib/components/Icons/IconAnyToAny.svelte new file mode 100644 index 000000000..9ebbf1b8c --- /dev/null +++ b/packages/widgets/src/lib/components/Icons/IconAnyToAny.svelte @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte index 02cb84f27..1429e1cb4 100644 --- a/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte +++ b/packages/widgets/src/lib/components/PipelineIcon/PipelineIcon.svelte @@ -43,6 +43,7 @@ import IconImageTo3D from "../Icons/IconImageTo3D.svelte"; import IconImageFeatureExtraction from "../Icons/IconImageFeatureExtraction.svelte"; import IconVideoTextToText from "../Icons/IconVideoTextToText.svelte"; + import IconAnyToAny from "../Icons/IconAnyToAny.svelte"; import IconKeypointDetection from "../Icons/IconKeypointDetection.svelte"; import type { WidgetType } from "@huggingface/tasks"; @@ -97,6 +98,7 @@ "image-to-3d": IconImageTo3D, "image-feature-extraction": IconImageFeatureExtraction, "video-text-to-text": IconVideoTextToText, + "any-to-any": IconAnyToAny, "keypoint-detection": IconKeypointDetection, };