Add route to get video details

cmd/subtle/main.go

@@ -21,6 +21,5 @@ func main() {
 	}
 
 	server := server.New()
-
 	server.Listen(3000, true)
 }

internal/routine/extract/extract.go

@@ -14,6 +14,7 @@ import (
 	"github.com/nandesh-dev/subtle/pkgs/filemanager"
 	"github.com/nandesh-dev/subtle/pkgs/pgs"
 	"github.com/nandesh-dev/subtle/pkgs/subtitle"
+	"github.com/nandesh-dev/subtle/pkgs/tesseract"
 	"github.com/nandesh-dev/subtle/pkgs/warning"
 	"golang.org/x/text/language"
 )
@@ -117,16 +118,15 @@ func extractSubtitleFromDirectory(dir filemanager.Directory, autoExtractConfig c
 		case subtitle.TextSubtitle:
 			subtitleEntry := db.Subtitle{
 				Language: rawStream.Language().String(),
-				Filepath: "",
-				IsImage:  false,
 				Segments: make([]db.Segment, 0),
 			}
 
 			for _, segment := range sub.Segments() {
 				segmentEntry := db.Segment{
-					StartTime: segment.Start(),
-					EndTime:   segment.End(),
-					Text:      segment.Text(),
+					StartTime:    segment.Start(),
+					EndTime:      segment.End(),
+					Text:         segment.Text(),
+					OriginalText: segment.Text(),
 				}
 
 				subtitleEntry.Segments = append(subtitleEntry.Segments, segmentEntry)
@@ -138,22 +138,30 @@ func extractSubtitleFromDirectory(dir filemanager.Directory, autoExtractConfig c
 		case subtitle.ImageSubtitle:
 			subtitleEntry := db.Subtitle{
 				Language: rawStream.Language().String(),
-				Filepath: "",
-				IsImage:  true,
 				Segments: make([]db.Segment, 0),
 			}
 
+			tesseractClient := tesseract.NewClient()
+			defer tesseractClient.Close()
+
 			for _, segment := range sub.Segments() {
 				imageDataBuffer := new(bytes.Buffer)
 				if err := png.Encode(imageDataBuffer, segment.Image()); err != nil {
 					warnings.AddWarning(fmt.Errorf("Error encoding image to png for video: %v; %v", video.Filepath(), err))
 					continue
 				}
 
+				text, err := tesseractClient.ExtractTextFromPNGImage(*imageDataBuffer, rawStream.Language())
+				if err != nil {
+					warnings.AddWarning(fmt.Errorf("Error extracting text from image: %v", err))
+				}
+
 				segmentEntry := db.Segment{
-					StartTime: segment.Start(),
-					EndTime:   segment.End(),
-					ImageData: imageDataBuffer.Bytes(),
+					StartTime:     segment.Start(),
+					EndTime:       segment.End(),
+					Text:          text,
+					OriginalText:  text,
+					OriginalImage: imageDataBuffer.Bytes(),
 				}
 
 				subtitleEntry.Segments = append(subtitleEntry.Segments, segmentEntry)

internal/server/media/get_video.go

@@ -0,0 +1,55 @@
+package media
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"connectrpc.com/connect"
+	"github.com/nandesh-dev/subtle/generated/proto/media"
+	"github.com/nandesh-dev/subtle/pkgs/db"
+	"github.com/nandesh-dev/subtle/pkgs/filemanager"
+	"github.com/nandesh-dev/subtle/pkgs/subtitle"
+)
+
+func (s ServiceHandler) GetVideo(ctx context.Context, req *connect.Request[media.GetVideoRequest]) (*connect.Response[media.GetVideoResponse], error) {
+	var videoEntry db.Video
+
+	if err := db.DB().Where(&db.Video{DirectoryPath: req.Msg.DirectoryPath, Filename: req.Msg.Name + req.Msg.Extension}).
+		Preload("Subtitles").
+		Preload("Subtitles.Segments").
+		First(&videoEntry).Error; err != nil {
+		return nil, fmt.Errorf("Error getting video entry: %v", err)
+	}
+
+	rawStreams, err := filemanager.NewVideoFile(filepath.Join(req.Msg.DirectoryPath, req.Msg.Name+req.Msg.Extension)).RawStreams()
+
+	if err != nil {
+		return nil, fmt.Errorf("Error extracting available raw stream from video: %v", err)
+	}
+
+	res := media.GetVideoResponse{
+		Subtitles:  make([]*media.Subtitle, len(videoEntry.Subtitles)),
+		RawStreams: make([]*media.RawStream, len(*rawStreams)),
+	}
+
+	for i, subtitleEntry := range videoEntry.Subtitles {
+		res.Subtitles[i] = &media.Subtitle{
+			Language:               subtitleEntry.Language,
+			ImportIsExternal:       subtitleEntry.ImportIsExternal,
+			ImportVideoStreamIndex: int32(subtitleEntry.ImportVideoStreamIndex),
+			ExportPath:             subtitleEntry.ExportPath,
+		}
+	}
+
+	for i, rawStream := range *rawStreams {
+		res.RawStreams[i] = &media.RawStream{
+			Index:    int32(rawStream.Index()),
+			Format:   subtitle.MapFormat(rawStream.Format()),
+			Language: rawStream.Language().String(),
+			Title:    rawStream.Title(),
+		}
+	}
+
+	return connect.NewResponse(&res), nil
+}

pkgs/db/db.go

@@ -25,9 +25,12 @@ type Subtitle struct {
 	VideoID int
 
 	Language string
-	Filepath string
-	IsImage  bool
 	Segments []Segment `gorm:"foreignKey:SubtitleID"`
+
+	ImportIsExternal       bool
+	ImportVideoStreamIndex int
+
+	ExportPath string
 }
 
 type Segment struct {
@@ -36,8 +39,10 @@ type Segment struct {
 
 	StartTime time.Duration
 	EndTime   time.Duration
-	ImageData []byte
 	Text      string
+
+	OriginalText  string
+	OriginalImage []byte
 }
 
 func DB() *gorm.DB {

pkgs/filemanager/video_file.go

@@ -9,6 +9,12 @@ type VideoFile struct {
 	path string
 }
 
+func NewVideoFile(path string) *VideoFile {
+	return &VideoFile{
+		path: path,
+	}
+}
+
 func (v *VideoFile) DirectoryPath() string {
 	return filepath.Dir(v.path)
 }

pkgs/subtitle/format.go

@@ -22,3 +22,16 @@ func ParseFormat(f string) (Format, error) {
 
 	return ASS, fmt.Errorf("Invalid format: %v", f)
 }
+
+func MapFormat(f Format) string {
+	switch f {
+	case ASS:
+		return "ass"
+	case PGS:
+		return "pgs"
+	case SRT:
+		return "srt"
+	}
+
+	return ""
+}

pkgs/tesseract/tesseract.go

@@ -44,3 +44,16 @@ func (c *TesseractClient) ExtractTextFromImage(img image.Image, lang language.Ta
 
 	return text, nil
 }
+
+func (c *TesseractClient) ExtractTextFromPNGImage(buf bytes.Buffer, lang language.Tag) (string, error) {
+	if err := c.gosseractClient.SetImageFromBytes(buf.Bytes()); err != nil {
+		return "", fmt.Errorf("Error scanning image: %v", err)
+	}
+
+	text, err := c.gosseractClient.Text()
+	if err != nil {
+		return "", fmt.Errorf("Error getting text from gosseract: %v", err)
+	}
+
+	return text, nil
+}

proto/media/media.proto

@@ -6,6 +6,34 @@ option go_package = "github.com/nandesh-dev/subtle/generated/proto/media";
 
 service MediaService {
   rpc GetDirectory(GetDirectoryRequest) returns (GetDirectoryResponse);
+  rpc GetVideo(GetVideoRequest) returns (GetVideoResponse);
+}
+
+message GetVideoRequest {
+  string directoryPath = 1;
+  string name = 2;
+  string extension = 3;
+}
+
+message GetVideoResponse {
+  repeated Subtitle subtitles = 1;
+  repeated RawStream raw_streams = 2;
+}
+
+message RawStream {
+  int32 index = 1;
+  string format = 2;
+  string language = 3;
+  string title = 4;
+}
+
+message Subtitle {
+  string language = 1;
+
+  bool importIsExternal = 2;
+  int32 importVideoStreamIndex = 3;
+
+  string exportPath = 4;
 }
 
 message GetDirectoryRequest {

web/src/main.tsx

@@ -11,6 +11,7 @@ import { createGrpcWebTransport } from '@connectrpc/connect-web'
 import { createClient } from '@connectrpc/connect'
 import { MediaService } from '../gen/proto/media/media_connect'
 import { QueryClient, QueryClientProvider } from '@tanstack/react-query'
+import { Video } from './routes/media/video/video'
 
 const router = createBrowserRouter([
     {
@@ -22,9 +23,13 @@ const router = createBrowserRouter([
                 element: <Home />,
             },
             {
-                path: 'media/*',
+                path: 'media',
                 element: <Media />,
             },
+            {
+                path: 'video',
+                element: <Video />,
+            },
         ],
     },
 ])

web/src/routes/media/media.tsx

@@ -1,4 +1,4 @@
-import { useSearchParams } from 'react-router-dom'
+import { useNavigate, useSearchParams } from 'react-router-dom'
 import { FolderIcon, SearchIcon } from '../../../assets'
 import { Large, Small } from '../../utils/react_responsive'
 import { useProto } from '../../context/proto'
@@ -46,6 +46,7 @@ export function Media() {
                         {data?.videos.map((video) => {
                             return (
                                 <File
+                                    directoryPath={path}
                                     key={video.name}
                                     name={video.name}
                                     extension={video.extension}
@@ -84,6 +85,7 @@ export function Media() {
                         {data?.videos.map((video) => {
                             return (
                                 <File
+                                    directoryPath={path}
                                     key={video.name}
                                     name={video.name}
                                     extension={video.extension}
@@ -133,23 +135,42 @@ function Folder({ name, subtitle, path }: FolderProp) {
 }
 
 type FileProp = {
+    directoryPath: string
     name: string
     extension: string
 }
 
-function File({ name, extension }: FileProp) {
+function File({ name, extension, directoryPath }: FileProp) {
+    const navigate = useNavigate()
+
+    const onClick = () => {
+        const newSearchParam = new URLSearchParams({
+            directoryPath,
+            name,
+            extension,
+        })
+
+        navigate('/video?' + newSearchParam.toString(), {})
+    }
+
     return (
         <>
             <Small>
-                <button className="grid grid-rows-2 gap-sm rounded-sm bg-gray-80 p-sm">
+                <button
+                    className="grid grid-rows-2 gap-sm rounded-sm bg-gray-80 p-sm"
+                    onClick={onClick}
+                >
                     <p className="text-start text-sm text-gray-830">{name}</p>
                     <div className="flex flex-row justify-between">
                         <p className="text-sm text-gray-520">{extension}</p>
                     </div>
                 </button>
             </Small>
             <Large>
-                <button className="grid grid-cols-2 rounded-sm bg-gray-80 p-sm">
+                <button
+                    className="grid grid-cols-2 rounded-sm bg-gray-80 p-sm"
+                    onClick={onClick}
+                >
                     <p className="text-start text-sm text-gray-830">{name}</p>
                     <div className="grid grid-cols-3">
                         <p className="text-sm text-gray-520">{extension}</p>