diff --git a/appinfo/info.xml b/appinfo/info.xml index 21e84814..8d70b9ae 100644 --- a/appinfo/info.xml +++ b/appinfo/info.xml @@ -34,7 +34,7 @@ include text processing providers to: * Get an answer from a free prompt * Reformulate (OpenAi/LocalAi only) ]]> - 1.0.3 + 1.0.5 agpl Julien Veyssier TpAssistant @@ -49,10 +49,11 @@ include text processing providers to: https://github.com/nextcloud/assistant/raw/main/img/screenshot3.jpg OCA\TpAssistant\Cron\CleanupImageGenerations - OCA\TpAssistant\Cron\CleanupTranscriptions + OCA\TpAssistant\Cron\CleanupAssistantTasks OCA\TpAssistant\Command\CleanupImageGenerations + OCA\TpAssistant\Command\CleanupAssistantTasks diff --git a/appinfo/routes.php b/appinfo/routes.php index b84df620..a2118c20 100644 --- a/appinfo/routes.php +++ b/appinfo/routes.php @@ -6,9 +6,12 @@ ['name' => 'config#setConfig', 'url' => '/config', 'verb' => 'PUT'], ['name' => 'config#setAdminConfig', 'url' => '/admin-config', 'verb' => 'PUT'], - ['name' => 'assistant#getTextProcessingTaskResultPage', 'url' => '/t/{taskId}', 'verb' => 'GET'], - ['name' => 'assistant#runTextProcessingTask', 'url' => '/run', 'verb' => 'POST'], - ['name' => 'assistant#runOrScheduleTextProcessingTask', 'url' => '/run-or-schedule', 'verb' => 'POST'], + ['name' => 'assistant#getTextProcessingTaskResultPage', 'url' => '/task/view/{taskId}', 'verb' => 'GET'], + ['name' => 'assistant#runTextProcessingTask', 'url' => '/task/run', 'verb' => 'POST'], + ['name' => 'assistant#scheduleTextProcessingTask', 'url' => '/task/schedule', 'verb' => 'POST'], + ['name' => 'assistant#runOrScheduleTextProcessingTask', 'url' => '/task/run-or-schedule', 'verb' => 'POST'], + ['name' => 'assistant#getTextProcessingResult', 'url' => '/task/{taskId}', 'verb' => 'GET'], + ['name' => 'assistant#parseTextFromFile', 'url' => '/parse-file', 'verb' => 'POST'], ['name' => 'Text2Image#processPrompt', 'url' => '/i/process_prompt', 'verb' => 'POST'], ['name' => 'Text2Image#getPromptHistory', 'url' => '/i/prompt_history', 'verb' => 'GET'], @@ -25,7 +28,6 @@ ['name' => 'FreePrompt#cancelGeneration', 'url' => '/f/cancel_generation', 'verb' => 'POST'], ['name' => 'SpeechToText#getResultPage', 'url' => '/stt/resultPage', 'verb' => 'GET'], - ['name' => 'SpeechToText#getTranscript', 'url' => '/stt/transcript', 'verb' => 'GET'], ['name' => 'SpeechToText#transcribeAudio', 'url' => '/stt/transcribeAudio', 'verb' => 'POST'], ['name' => 'SpeechToText#transcribeFile', 'url' => '/stt/transcribeFile', 'verb' => 'POST'], ], diff --git a/composer.json b/composer.json index d602b616..4d39d301 100644 --- a/composer.json +++ b/composer.json @@ -11,7 +11,9 @@ } ], "require": { - "php": "^8.0" + "php": "^8.0", + "erusev/parsedown": "^1.7", + "phpoffice/phpword": "^1.2" }, "scripts": { "lint": "find . -name \\*.php -not -path './vendor/*' -print0 | xargs -0 -n1 php -l", @@ -25,5 +27,13 @@ "psalm/phar": "^5.16", "nextcloud/ocp": "dev-master", "phpunit/phpunit": "^9.5" + }, + "config": { + "sort-packages": true, + "optimize-autoloader": true, + "platform": { + "php": "8.0" + }, + "autoloader-suffix": "TpAssistant" } } diff --git a/lib/AppInfo/Application.php b/lib/AppInfo/Application.php index 636fedf0..43af1ff6 100644 --- a/lib/AppInfo/Application.php +++ b/lib/AppInfo/Application.php @@ -32,6 +32,7 @@ class Application extends App implements IBootstrap { public const APP_ID = 'assistant'; + public const DEFAULT_ASSISTANT_TASK_IDLE_TIME = 60 * 60 * 24 * 14; // 14 days public const MAX_STORED_IMAGE_PROMPTS_PER_USER = 5; public const MAX_STORED_TEXT_PROMPTS_PER_USER = 5; @@ -40,9 +41,13 @@ class Application extends App implements IBootstrap { public const IMAGE_FOLDER = 'generated_images'; public const SPEECH_TO_TEXT_REC_FOLDER = 'stt_recordings'; - public const TASK_TYPE_TEXT_GEN = 0; - public const TASK_TYPE_TEXT_TO_IMAGE = 1; - public const TASK_TYPE_SPEECH_TO_TEXT = 2; + public const STT_TASK_SCHEDULED = 0; + public const STT_TASK_SUCCESSFUL = 1; + public const STT_TASK_FAILED = -1; + + public const TASK_CATEGORY_TEXT_GEN = 0; + public const TASK_CATEGORY_TEXT_TO_IMAGE = 1; + public const TASK_CATEGORY_SPEECH_TO_TEXT = 2; public function __construct(array $urlParams = []) { parent::__construct(self::APP_ID, $urlParams); diff --git a/lib/Command/CleanupAssistantTasks.php b/lib/Command/CleanupAssistantTasks.php new file mode 100644 index 00000000..566d34cb --- /dev/null +++ b/lib/Command/CleanupAssistantTasks.php @@ -0,0 +1,54 @@ + +// SPDX-License-Identifier: AGPL-3.0-or-later + +namespace OCA\TpAssistant\Command; + +use Exception; +use OC\Core\Command\Base; +use OCA\TpAssistant\AppInfo\Application; +use OCA\TpAssistant\Db\MetaTaskMapper; +use Symfony\Component\Console\Input\InputArgument; +use Symfony\Component\Console\Input\InputInterface; +use Symfony\Component\Console\Output\OutputInterface; + +class CleanupAssistantTasks extends Base { + public function __construct( + private MetaTaskMapper $metaTaskMapper, + ) { + parent::__construct(); + } + + protected function configure() { + $maxIdleTimeSetting = Application::DEFAULT_ASSISTANT_TASK_IDLE_TIME; + $this->setName('assistant:task_cleanup') + ->setDescription('Cleanup assistant tasks') + ->addArgument( + 'max_age', + InputArgument::OPTIONAL, + 'The max idle time (in seconds)', + $maxIdleTimeSetting + ); + } + + protected function execute(InputInterface $input, OutputInterface $output) { + $maxAge = intval($input->getArgument('max_age')); + + if ($maxAge < 1) { + $output->writeln('Invalid value for max_age: ' . $maxAge); + return 1; + } + + $output->writeln('Cleanning up assistant tasks older than ' . $maxAge . ' seconds.'); + try { + $cleanedUp = $this->metaTaskMapper->cleanupOldMetaTasks($maxAge); + } catch (Exception $e) { + $output->writeln('Error: ' . $e->getMessage()); + return 1; + } + + $output->writeln('Deleted ' . $cleanedUp . ' idle tasks.'); + return 0; + } +} diff --git a/lib/Controller/AssistantController.php b/lib/Controller/AssistantController.php index 4162667d..c92daa37 100644 --- a/lib/Controller/AssistantController.php +++ b/lib/Controller/AssistantController.php @@ -6,23 +6,21 @@ use OCA\TpAssistant\Service\AssistantService; use OCP\AppFramework\Controller; use OCP\AppFramework\Http; -use OCP\AppFramework\Http\Attribute\BruteForceProtection; use OCP\AppFramework\Http\Attribute\NoAdminRequired; use OCP\AppFramework\Http\Attribute\NoCSRFRequired; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\TemplateResponse; use OCP\AppFramework\Services\IInitialState; - use OCP\IRequest; class AssistantController extends Controller { public function __construct( - string $appName, - IRequest $request, + string $appName, + IRequest $request, private AssistantService $assistantService, - private IInitialState $initialStateService, - private ?string $userId + private IInitialState $initialStateService, + private ?string $userId, ) { parent::__construct($appName, $request); } @@ -33,59 +31,124 @@ public function __construct( */ #[NoAdminRequired] #[NoCSRFRequired] - #[BruteForceProtection(action: 'taskResultPage')] public function getTextProcessingTaskResultPage(int $taskId): TemplateResponse { - $task = $this->assistantService->getTextProcessingTask($this->userId, $taskId); - if ($task === null) { - $response = new TemplateResponse( - '', - '403', - [], - TemplateResponse::RENDER_AS_ERROR - ); - $response->setStatus(Http::STATUS_NOT_FOUND); - $response->throttle(['userId' => $this->userId, 'taskId' => $taskId]); - return $response; + + if ($this->userId !== null) { + $task = $this->assistantService->getTextProcessingTask($this->userId, $taskId); + if ($task !== null) { + $this->initialStateService->provideInitialState('task', $task->jsonSerializeCc()); + return new TemplateResponse(Application::APP_ID, 'taskResultPage'); + } + } + return new TemplateResponse('', '403', [], TemplateResponse::RENDER_AS_ERROR, Http::STATUS_FORBIDDEN); + } + + /** + * @param int $taskId + * @return DataResponse + */ + #[NoAdminRequired] + public function getTextProcessingResult(int $taskId): DataResponse { + + if ($this->userId !== null) { + $task = $this->assistantService->getTextProcessingTask($this->userId, $taskId); + if ($task !== null) { + return new DataResponse([ + 'task' => $task->jsonSerializeCc(), + ]); + } + } + return new DataResponse('', Http::STATUS_NOT_FOUND); + } + + /** + * @param array $inputs + * @param string $type + * @param string $appId + * @param string $identifier + * @return DataResponse + */ + #[NoAdminRequired] + public function runTextProcessingTask(string $type, array $inputs, string $appId, string $identifier): DataResponse { + if ($this->userId === null) { + return new DataResponse('Unknow user', Http::STATUS_BAD_REQUEST); + } + + try { + $task = $this->assistantService->runTextProcessingTask($type, $inputs, $appId, $this->userId, $identifier); + } catch (\Exception | \Throwable $e) { + return new DataResponse($e->getMessage(), Http::STATUS_BAD_REQUEST); } - $this->initialStateService->provideInitialState('task', $task->jsonSerialize()); - return new TemplateResponse(Application::APP_ID, 'taskResultPage'); + return new DataResponse([ + 'task' => $task->jsonSerializeCc(), + ]); } /** - * @param string $input + * @param array $inputs * @param string $type * @param string $appId * @param string $identifier * @return DataResponse */ #[NoAdminRequired] - public function runTextProcessingTask(string $type, string $input, string $appId, string $identifier): DataResponse { + public function scheduleTextProcessingTask(string $type, array $inputs, string $appId, string $identifier): DataResponse { + if ($this->userId === null) { + return new DataResponse('Unknow user', Http::STATUS_BAD_REQUEST); + } + try { - $task = $this->assistantService->runTextProcessingTask($type, $input, $appId, $this->userId, $identifier); + $task = $this->assistantService->scheduleTextProcessingTask($type, $inputs, $appId, $this->userId, $identifier); } catch (\Exception | \Throwable $e) { return new DataResponse($e->getMessage(), Http::STATUS_BAD_REQUEST); } return new DataResponse([ - 'task' => $task->jsonSerialize(), + 'task' => $task->jsonSerializeCc(), ]); } /** - * @param string $input + * @param array $inputs * @param string $type * @param string $appId * @param string $identifier * @return DataResponse */ #[NoAdminRequired] - public function runOrScheduleTextProcessingTask(string $type, string $input, string $appId, string $identifier): DataResponse { + public function runOrScheduleTextProcessingTask(string $type, array $inputs, string $appId, string $identifier): DataResponse { + if ($this->userId === null) { + return new DataResponse('Unknow user', Http::STATUS_BAD_REQUEST); + } + + try { + $task = $this->assistantService->runOrScheduleTextProcessingTask($type, $inputs, $appId, $this->userId, $identifier); + } catch (\Exception | \Throwable $e) { + return new DataResponse($e->getMessage(), Http::STATUS_BAD_REQUEST); + } + return new DataResponse([ + 'task' => $task->jsonSerializeCc(), + ]); + } + + /** + * Parse text from file (if parsing the file type is supported) + * + * @param string $filePath + * @return DataResponse + */ + #[NoAdminRequired] + public function parseTextFromFile(string $filePath): DataResponse { + if ($this->userId === null) { + return new DataResponse('Unknow user', Http::STATUS_BAD_REQUEST); + } + try { - $task = $this->assistantService->runOrScheduleTextProcessingTask($type, $input, $appId, $this->userId, $identifier); + $text = $this->assistantService->parseTextFromFile($filePath, $this->userId); } catch (\Exception | \Throwable $e) { return new DataResponse($e->getMessage(), Http::STATUS_BAD_REQUEST); } return new DataResponse([ - 'task' => $task->jsonSerialize(), + 'parsedText' => $text, ]); } } diff --git a/lib/Controller/FreePromptController.php b/lib/Controller/FreePromptController.php index ee0305e6..f35d5358 100644 --- a/lib/Controller/FreePromptController.php +++ b/lib/Controller/FreePromptController.php @@ -12,8 +12,9 @@ use OCP\AppFramework\Http\Attribute\NoAdminRequired; use OCP\AppFramework\Http\Attribute\NoCSRFRequired; use OCP\AppFramework\Http\DataResponse; - use OCP\AppFramework\Services\IInitialState; + +use OCP\IL10N; use OCP\IRequest; class FreePromptController extends Controller { @@ -22,7 +23,8 @@ public function __construct( IRequest $request, private FreePromptService $freePromptService, private ?string $userId, - private IInitialState $initialStateService + private IInitialState $initialStateService, + private IL10N $l10n, ) { parent::__construct($appName, $request); } @@ -33,9 +35,14 @@ public function __construct( */ #[NoAdminRequired] #[NoCSRFRequired] - public function processPrompt(string $prompt, int $nResults = 1): DataResponse { + public function processPrompt(string $prompt): DataResponse { + + if ($this->userId === null) { + return new DataResponse(['error' => $this->l10n->t('Failed to process prompt; unknown user')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + try { - $result = $this->freePromptService->processPrompt($prompt, $nResults); + $result = $this->freePromptService->processPrompt($prompt, $this->userId); } catch (Exception $e) { return new DataResponse(['error' => $e->getMessage()], (int)$e->getCode()); } @@ -49,8 +56,13 @@ public function processPrompt(string $prompt, int $nResults = 1): DataResponse { #[NoAdminRequired] #[NoCSRFRequired] public function getPromptHistory(): DataResponse { + + if ($this->userId === null) { + return new DataResponse(['error' => $this->l10n->t('Failed to get prompt history; unknown user')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + try { - $result = $this->freePromptService->getPromptHistory(); + $result = $this->freePromptService->getPromptHistory($this->userId); } catch (Exception $e) { return new DataResponse(['error' => $e->getMessage()], (int)$e->getCode()); } @@ -66,8 +78,13 @@ public function getPromptHistory(): DataResponse { #[NoAdminRequired] #[NoCSRFRequired] public function getOutputs(string $genId): DataResponse { + + if ($this->userId === null) { + return new DataResponse(['error' => $this->l10n->t('Failed to get outputs; unknown user')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + try { - $result = $this->freePromptService->getOutputs($genId); + $result = $this->freePromptService->getOutputs($genId, $this->userId); } catch (Exception $e) { return new DataResponse(['error' => $e->getMessage()], (int)$e->getCode()); } @@ -83,8 +100,13 @@ public function getOutputs(string $genId): DataResponse { #[NoAdminRequired] #[NoCSRFRequired] public function cancelGeneration(string $genId): DataResponse { + + if ($this->userId === null) { + return new DataResponse(['error' => $this->l10n->t('Failed to cancel generation; unknown user')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + try { - $this->freePromptService->cancelGeneration($genId); + $this->freePromptService->cancelGeneration($genId, $this->userId); } catch (Exception $e) { $response = new DataResponse(['error' => $e->getMessage()], (int)$e->getCode()); return $response; diff --git a/lib/Controller/SpeechToTextController.php b/lib/Controller/SpeechToTextController.php index 3731089b..79deb1ae 100644 --- a/lib/Controller/SpeechToTextController.php +++ b/lib/Controller/SpeechToTextController.php @@ -22,20 +22,18 @@ namespace OCA\TpAssistant\Controller; -use DateTime; use Exception; use InvalidArgumentException; use OCA\TpAssistant\AppInfo\Application; -use OCA\TpAssistant\Db\SpeechToText\TranscriptMapper; +use OCA\TpAssistant\Db\MetaTask; +use OCA\TpAssistant\Db\MetaTaskMapper; use OCA\TpAssistant\Service\SpeechToText\SpeechToTextService; use OCP\AppFramework\Controller; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\AppFramework\Http; -use OCP\AppFramework\Http\Attribute\AnonRateLimit; use OCP\AppFramework\Http\Attribute\NoAdminRequired; use OCP\AppFramework\Http\Attribute\NoCSRFRequired; -use OCP\AppFramework\Http\Attribute\UserRateLimit; use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\TemplateResponse; use OCP\AppFramework\Services\IInitialState; @@ -50,14 +48,14 @@ class SpeechToTextController extends Controller { public function __construct( - string $appName, - IRequest $request, + string $appName, + IRequest $request, private SpeechToTextService $service, - private LoggerInterface $logger, - private IL10N $l10n, - private TranscriptMapper $transcriptMapper, - private IInitialState $initialState, - private ?string $userId, + private LoggerInterface $logger, + private IL10N $l10n, + private IInitialState $initialState, + private ?string $userId, + private MetaTaskMapper $metaTaskMapper, ) { parent::__construct($appName, $request); } @@ -68,19 +66,16 @@ public function __construct( */ #[NoAdminRequired] #[NoCSRFRequired] - #[UserRateLimit(limit: 10, period: 60)] - #[AnonRateLimit(limit: 2, period: 60)] public function getResultPage(int $id): TemplateResponse { $response = new TemplateResponse(Application::APP_ID, 'speechToTextResultPage'); try { $initData = [ - 'status' => 'success', - 'result' => $this->internalGetTranscript($id), - 'taskType' => Application::TASK_TYPE_SPEECH_TO_TEXT, + 'task' => $this->internalGetTask($id), ]; } catch (Exception $e) { $initData = [ 'status' => 'failure', + 'task' => null, 'message' => $e->getMessage(), ]; $response->setStatus(intval($e->getCode())); @@ -96,39 +91,30 @@ public function getResultPage(int $id): TemplateResponse { #[NoAdminRequired] public function getTranscript(int $id): DataResponse { try { - return new DataResponse($this->internalGetTranscript($id)); + return new DataResponse($this->internalGetTask($id)->getOutput()); } catch (Exception $e) { return new DataResponse($e->getMessage(), intval($e->getCode())); } } /** - * Internal function to get transcript and throw a common exception + * Internal function to get transcription assistant tasks based on the assistant meta task id * * @param integer $id - * @return string + * @return MetaTask */ - private function internalGetTranscript(int $id): string { + private function internalGetTask(int $id): MetaTask { try { - $transcriptEntity = $this->transcriptMapper->find($id, $this->userId); - $transcript = $transcriptEntity->getTranscript(); + $metaTask = $this->metaTaskMapper->getMetaTaskOfUser($id, $this->userId); - $transcriptEntity->setLastAccessed(new DateTime()); - $this->transcriptMapper->update($transcriptEntity); + if($metaTask->getCategory() !== Application::TASK_CATEGORY_SPEECH_TO_TEXT) { + throw new Exception('Task is not a speech to text task.', Http::STATUS_BAD_REQUEST); + } - return trim($transcript); - } catch (InvalidArgumentException $e) { - $this->logger->error( - 'Invalid argument in transcript access time update call: ' . $e->getMessage(), - ['app' => Application::APP_ID], - ); - throw new Exception( - $this->l10n->t('Error in transcript access time update call'), - Http::STATUS_INTERNAL_SERVER_ERROR, - ); + return $metaTask; } catch (MultipleObjectsReturnedException $e) { - $this->logger->error('Multiple transcripts found: ' . $e->getMessage(), ['app' => Application::APP_ID]); - throw new Exception($this->l10n->t('Multiple transcripts found'), Http::STATUS_BAD_REQUEST); + $this->logger->error('Multiple tasks found for one id: ' . $e->getMessage(), ['app' => Application::APP_ID]); + throw new Exception($this->l10n->t('Multiple tasks found'), Http::STATUS_BAD_REQUEST); } catch (DoesNotExistException $e) { throw new Exception($this->l10n->t('Transcript not found'), Http::STATUS_NOT_FOUND); } catch (Exception $e) { diff --git a/lib/Controller/Text2ImageController.php b/lib/Controller/Text2ImageController.php index 1f49f626..5493f6f2 100644 --- a/lib/Controller/Text2ImageController.php +++ b/lib/Controller/Text2ImageController.php @@ -19,8 +19,9 @@ use OCP\AppFramework\Http\DataResponse; use OCP\AppFramework\Http\TemplateResponse; use OCP\AppFramework\Services\IInitialState; - use OCP\Db\Exception as DbException; + +use OCP\IL10N; use OCP\IRequest; use OCP\TextToImage\Exception\TaskFailureException; @@ -30,7 +31,8 @@ public function __construct( IRequest $request, private Text2ImageHelperService $text2ImageHelperService, private IInitialState $initialStateService, - private ?string $userId + private ?string $userId, + private IL10N $l10n, ) { parent::__construct($appName, $request); } @@ -46,7 +48,7 @@ public function __construct( public function processPrompt(string $prompt, int $nResults = 1, bool $displayPrompt = false): DataResponse { $nResults = min(10, max(1, $nResults)); try { - $result = $this->text2ImageHelperService->processPrompt($prompt, $nResults, $displayPrompt); + $result = $this->text2ImageHelperService->processPrompt($prompt, $nResults, $displayPrompt, $this->userId); } catch (Exception | TaskFailureException $e) { return new DataResponse(['error' => $e->getMessage()], Http::STATUS_BAD_REQUEST); } @@ -60,10 +62,15 @@ public function processPrompt(string $prompt, int $nResults = 1, bool $displayPr #[NoAdminRequired] #[NoCSRFRequired] public function getPromptHistory(): DataResponse { + + if ($this->userId === null) { + return new DataResponse(['error' => $this->l10n->t('Failed to get prompt history; unknown user')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + try { - $response = $this->text2ImageHelperService->getPromptHistory(); + $response = $this->text2ImageHelperService->getPromptHistory($this->userId); } catch (DbException $e) { - return new DataResponse(['error' => 'Unknown error while retrieving prompt history.'], Http::STATUS_INTERNAL_SERVER_ERROR); + return new DataResponse(['error' => $this->l10n->t('Unknown error while retrieving prompt history.')], Http::STATUS_INTERNAL_SERVER_ERROR); } return new DataResponse($response); @@ -79,7 +86,6 @@ public function getPromptHistory(): DataResponse { #[PublicPage] #[BruteForceProtection(action: 'imageGenId')] public function getImage(string $imageGenId, int $fileNameId): DataDisplayResponse | DataResponse { - try { $result = $this->text2ImageHelperService->getImage($imageGenId, $fileNameId); } catch (Exception $e) { @@ -91,9 +97,11 @@ public function getImage(string $imageGenId, int $fileNameId): DataDisplayRespon return $response; } + /* if (isset($result['processing'])) { return new DataResponse($result, Http::STATUS_OK); } + */ return new DataDisplayResponse( $result['image'] ?? '', @@ -112,7 +120,7 @@ public function getImage(string $imageGenId, int $fileNameId): DataDisplayRespon #[BruteForceProtection(action: 'imageGenId')] public function getGenerationInfo(string $imageGenId): DataResponse { try { - $result = $this->text2ImageHelperService->getGenerationInfo($imageGenId, true); + $result = $this->text2ImageHelperService->getGenerationInfo($imageGenId, $this->userId, true); } catch (Exception $e) { $response = new DataResponse(['error' => $e->getMessage()], (int) $e->getCode()); if ($e->getCode() === Http::STATUS_BAD_REQUEST || $e->getCode() === Http::STATUS_UNAUTHORIZED) { @@ -133,12 +141,17 @@ public function getGenerationInfo(string $imageGenId): DataResponse { #[NoCSRFRequired] #[BruteForceProtection(action: 'imageGenId')] public function setVisibilityOfImageFiles(string $imageGenId, array $fileVisStatusArray): DataResponse { + + if ($this->userId === null) { + return new DataResponse(['error' => $this->l10n->t('Failed to set visibility of image files; unknown user')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + if (count($fileVisStatusArray) < 1) { return new DataResponse('File visibility array empty', Http::STATUS_BAD_REQUEST); } try { - $this->text2ImageHelperService->setVisibilityOfImageFiles($imageGenId, $fileVisStatusArray); + $this->text2ImageHelperService->setVisibilityOfImageFiles($imageGenId, $fileVisStatusArray, $this->userId); } catch (Exception $e) { $response = new DataResponse(['error' => $e->getMessage()], (int) $e->getCode()); if($e->getCode() === Http::STATUS_BAD_REQUEST || $e->getCode() === Http::STATUS_UNAUTHORIZED) { @@ -162,8 +175,13 @@ public function setVisibilityOfImageFiles(string $imageGenId, array $fileVisStat #[NoCSRFRequired] #[AnonRateLimit(limit: 10, period: 60)] public function notifyWhenReady(string $imageGenId): DataResponse { + + if ($this->userId === null) { + return new DataResponse(['error' => $this->l10n->t('Failed to notify when ready; unknown user')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + try { - $this->text2ImageHelperService->notifyWhenReady($imageGenId); + $this->text2ImageHelperService->notifyWhenReady($imageGenId, $this->userId); } catch (Exception $e) { // Ignore } @@ -183,7 +201,12 @@ public function notifyWhenReady(string $imageGenId): DataResponse { #[NoCSRFRequired] #[AnonRateLimit(limit: 10, period: 60)] public function cancelGeneration(string $imageGenId): DataResponse { - $this->text2ImageHelperService->cancelGeneration($imageGenId); + + if ($this->userId === null) { + return new DataResponse(['error' => $this->l10n->t('Failed to cancel generation; unknown user')], Http::STATUS_INTERNAL_SERVER_ERROR); + } + + $this->text2ImageHelperService->cancelGeneration($imageGenId, $this->userId); return new DataResponse('success', Http::STATUS_OK); } @@ -202,12 +225,8 @@ public function showGenerationPage(?string $imageGenId, ?bool $forceEditMode = f if ($forceEditMode === null) { $forceEditMode = false; } - if ($imageGenId === null) { - $this->initialStateService->provideInitialState('generation-page-inputs', ['image_gen_id' => $imageGenId, 'force_edit_mode' => $forceEditMode]); - } else { - $this->initialStateService->provideInitialState('generation-page-inputs', ['image_gen_id' => $imageGenId, 'force_edit_mode' => $forceEditMode]); - } - + $this->initialStateService->provideInitialState('generation-page-inputs', ['image_gen_id' => $imageGenId, 'force_edit_mode' => $forceEditMode]); + return new TemplateResponse(Application::APP_ID, 'imageGenerationPage'); } } diff --git a/lib/Cron/CleanupAssistantTasks.php b/lib/Cron/CleanupAssistantTasks.php new file mode 100644 index 00000000..288cdb7b --- /dev/null +++ b/lib/Cron/CleanupAssistantTasks.php @@ -0,0 +1,35 @@ + +// SPDX-License-Identifier: AGPL-3.0-or-later + +namespace OCA\TpAssistant\Cron; + +use Exception; +use OCA\TpAssistant\Db\MetaTaskMapper; +use OCP\AppFramework\Utility\ITimeFactory; +use OCP\BackgroundJob\TimedJob; +use Psr\Log\LoggerInterface; +use RuntimeException; + +class CleanupAssistantTasks extends TimedJob { + public function __construct( + ITimeFactory $time, + private LoggerInterface $logger, + private MetaTaskMapper $metaTaskMapper, + ) { + parent::__construct($time); + $this->setInterval(60 * 60 * 24); + } + + protected function run($argument): void { + $this->logger->debug('Run cleanup job for assistant tasks'); + + try { + $this->metaTaskMapper->cleanupOldMetaTasks(); + } catch (\OCP\Db\Exception | RuntimeException | Exception $e) { + $this->logger->debug('Cleanup job for assistant tasks failed: ' . $e->getMessage()); + } + } +} diff --git a/lib/Cron/CleanupImageGenerations.php b/lib/Cron/CleanupImageGenerations.php index 600b2465..961bccf6 100644 --- a/lib/Cron/CleanupImageGenerations.php +++ b/lib/Cron/CleanupImageGenerations.php @@ -7,7 +7,6 @@ namespace OCA\TpAssistant\Cron; use Exception; -use OCA\TpAssistant\Db\Text2Image\ImageGenerationMapper; use OCA\TpAssistant\Service\Text2Image\CleanUpService; use OCP\AppFramework\Utility\ITimeFactory; use OCP\BackgroundJob\TimedJob; @@ -16,9 +15,8 @@ class CleanupImageGenerations extends TimedJob { public function __construct( ITimeFactory $time, - private ImageGenerationMapper $imageGenerationMapper, + private CleanUpService $cleanUpService, private LoggerInterface $logger, - private CleanUpService $cleanUpService ) { parent::__construct($time); $this->setInterval(60 * 60 * 24); @@ -32,8 +30,5 @@ protected function run($argument): void { } catch (Exception $e) { $this->logger->debug('Cleanup job for image generations failed: ' . $e->getMessage()); } - - - return; } } diff --git a/lib/Cron/CleanupTranscriptions.php b/lib/Cron/CleanupTranscriptions.php deleted file mode 100644 index 5569d9fe..00000000 --- a/lib/Cron/CleanupTranscriptions.php +++ /dev/null @@ -1,45 +0,0 @@ - - * - * @author Anupam Kumar - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -namespace OCA\TpAssistant\Cron; - -use OCA\TpAssistant\Db\SpeechToText\TranscriptMapper; -use OCP\AppFramework\Utility\ITimeFactory; -use OCP\BackgroundJob\TimedJob; - -class CleanupTranscriptions extends TimedJob { - public function __construct( - ITimeFactory $time, - private TranscriptMapper $transcriptMapper, - ) { - parent::__construct($time); - $this->setInterval(60 * 60 * 24); // 24 hours - } - - protected function run($argument) { - $this->transcriptMapper->cleanupTranscriptions(); - } -} diff --git a/lib/Db/FreePrompt/Prompt.php b/lib/Db/FreePrompt/Prompt.php index af2a6cdb..4b3aa643 100644 --- a/lib/Db/FreePrompt/Prompt.php +++ b/lib/Db/FreePrompt/Prompt.php @@ -9,12 +9,12 @@ use OCP\AppFramework\Db\Entity; /** - * @method string getUserId() - * @method void setUserId(string $userId) - * @method string getValue() - * @method void setValue(string $value) - * @method int getTimestamp() - * @method void setTimestamp(int $timestamp) + * @method \string getUserId() + * @method \void setUserId(string $userId) + * @method \string getValue() + * @method \void setValue(string $value) + * @method \int getTimestamp() + * @method \void setTimestamp(int $timestamp) */ class Prompt extends Entity implements \JsonSerializable { /** @var string */ diff --git a/lib/Db/MetaTask.php b/lib/Db/MetaTask.php new file mode 100644 index 00000000..30d3592c --- /dev/null +++ b/lib/Db/MetaTask.php @@ -0,0 +1,108 @@ + +// SPDX-License-Identifier: AGPL-3.0-or-later + +namespace OCA\TpAssistant\Db; + +use OCP\AppFramework\Db\Entity; + +/** + * @method \string getUserId() + * @method \void setUserId(string $userId) + * @method \string getOutput() + * @method \void setOutput(string $value) + * @method \string getAppId() + * @method \void setAppId(string $appId) + * @method \int getOcpTaskId() + * @method \void setOcpTaskId(int $value) + * @method \int getTimestamp() + * @method \void setTimestamp(int $timestamp) + * @method \string getTaskType() + * @method \void setTaskType(string $taskType) + * @method \void setStatus(int $status) + * @method \int getStatus() + * @method \void setCategory(int $category) + * @method \int getCategory() + * @method \string getInputs() + * @method \void setInputs(string $inputs) + * @method \string getIdentifier() + * @method \void setIdentifier(string $identifier) + */ +class MetaTask extends Entity implements \JsonSerializable { + /** @var string */ + protected $userId; + /** @var string */ + protected $inputs; + /** @var string */ + protected $output; + /** @var string */ + protected $appId; + /** @var int */ + protected $ocpTaskId; + /** @var int */ + protected $timestamp; + /** @var string */ + protected $taskType; + /** @var int */ + protected $status; + /** @var int */ + protected $category; + /** @var string */ + protected $identifier; + + public function __construct() { + $this->addType('user_id', 'string'); + $this->addType('inputs', 'string'); + $this->addType('output', 'string'); + $this->addType('app_id', 'string'); + $this->addType('ocp_task_id', 'integer'); + $this->addType('timestamp', 'integer'); + $this->addType('task_type', 'string'); + $this->addType('status', 'integer'); + $this->addType('category', 'integer'); + $this->addType('identifier', 'string'); + } + + #[\ReturnTypeWillChange] + public function jsonSerialize() { + return [ + 'id' => $this->id, + 'user_id' => $this->userId, + 'inputs' => $this->getInputsAsArray(), + 'output' => $this->output, + 'app_id' => $this->appId, + 'ocp_task_id' => $this->ocpTaskId, + 'task_type' => $this->taskType, + 'timestamp' => $this->timestamp, + 'status' => $this->status, + 'category' => $this->category, + 'identifier' => $this->identifier, + ]; + } + + #[\ReturnTypeWillChange] + public function jsonSerializeCc() { + return [ + 'id' => $this->id, + 'userId' => $this->userId, + 'inputs' => $this->getInputsAsArray(), + 'output' => $this->output, + 'appId' => $this->appId, + 'ocpTaskId' => $this->ocpTaskId, + 'taskType' => $this->taskType, + 'timestamp' => $this->timestamp, + 'status' => $this->status, + 'category' => $this->category, + 'identifier' => $this->identifier, + ]; + } + + /** + * @return array + */ + public function getInputsAsArray(): array { + return json_decode($this->inputs, true) ?? []; + } +} diff --git a/lib/Db/MetaTaskMapper.php b/lib/Db/MetaTaskMapper.php new file mode 100644 index 00000000..d846db57 --- /dev/null +++ b/lib/Db/MetaTaskMapper.php @@ -0,0 +1,221 @@ + +// SPDX-License-Identifier: AGPL-3.0-or-later + +declare(strict_types=1); + +namespace OCA\TpAssistant\Db; + +use DateTime; +use OCA\TpAssistant\AppInfo\Application; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; +use OCP\AppFramework\Db\QBMapper; +use OCP\DB\Exception; +use OCP\DB\QueryBuilder\IQueryBuilder; +use OCP\IDBConnection; + +/** + * @extends QBMapper + */ +class MetaTaskMapper extends QBMapper { + public function __construct(IDBConnection $db) { + parent::__construct($db, 'assistant_meta_tasks', MetaTask::class); + } + + /** + * @param int $id + * @return MetaTask + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + */ + public function getMetaTask(int $id): MetaTask { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)) + ); + + return $this->findEntity($qb); + } + + /** + * @param int $ocpTaskId + * @param int $category + * @return MetaTask + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + */ + public function getMetaTaskByOcpTaskIdAndCategory(int $ocpTaskId, int $category): MetaTask { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('ocp_task_id', $qb->createNamedParameter($ocpTaskId, IQueryBuilder::PARAM_INT)) + ) + ->andWhere( + $qb->expr()->eq('category', $qb->createNamedParameter($category, IQueryBuilder::PARAM_INT)) + ); + + $retVal = $this->findEntity($qb); + + // Touch the timestamp to prevent the task from being cleaned up: + $retVal->setTimestamp((new DateTime())->getTimestamp()); + try { + $retVal = $this->update($retVal); + } catch (\InvalidArgumentException $e) { + // This should never happen + throw new Exception('Failed to touch timestamp of task', 0, $e); + } + + return $retVal; + } + + /** + * @param int $ocpTaskId + * @param int $category + * @return array + * @throws Exception + */ + public function getMetaTasksByOcpTaskIdAndCategory(int $ocpTaskId, int $category): array { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('ocp_task_id', $qb->createNamedParameter($ocpTaskId, IQueryBuilder::PARAM_INT)) + ) + ->andWhere( + $qb->expr()->eq('category', $qb->createNamedParameter($category, IQueryBuilder::PARAM_INT)) + ); + + $retVal = $this->findEntities($qb); + + // Touch the timestamps to prevent the task from being cleaned up: + foreach ($retVal as &$task) { + $task->setTimestamp((new DateTime())->getTimestamp()); + try { + $task = $this->update($task); + } catch (\InvalidArgumentException $e) { + // This should never happen + throw new Exception('Failed to touch timestamp of task', 0, $e); + } + } + unset($task); + + return $retVal; + } + + /** + * @param int $id + * @param string $userId + * @return MetaTask + * @throws DoesNotExistException + * @throws Exception + * @throws MultipleObjectsReturnedException + */ + public function getMetaTaskOfUser(int $id, string $userId): MetaTask { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('id', $qb->createNamedParameter($id, IQueryBuilder::PARAM_INT)) + ) + ->andWhere( + $qb->expr()->eq('user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR)) + ); + return $this->findEntity($qb); + } + + /** + * @param string $userId + * @return array + * @throws Exception + */ + public function getMetaTasksOfUser(string $userId): array { + $qb = $this->db->getQueryBuilder(); + + $qb->select('*') + ->from($this->getTableName()) + ->where( + $qb->expr()->eq('user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR)) + ); + + return $this->findEntities($qb); + } + + /** + * @param string $userId + * @param array $inputs + * @param string|null $output + * @param int|null $timestamp + * @param int|null $ocpTaskId + * @param string|null $taskType + * @param string|null $appId + * @param int $status + * @param int $category + * @param string $identifier + * @return MetaTask + * @throws Exception + */ + public function createMetaTask( + string $userId, array $inputs, ?string $output, ?int $timestamp = null, ?int $ocpTaskId = null, + ?string $taskType = null, ?string $appId = null, int $status = 0, int $category = 0, string $identifier = '' + ): MetaTask { + if ($timestamp === null) { + $timestamp = (new DateTime())->getTimestamp(); + } + + $task = new MetaTask(); + $task->setUserId($userId); + $task->setInputs(json_encode($inputs)); + $task->setTimestamp($timestamp); + $task->setOutput($output); + $task->setOcpTaskId($ocpTaskId); + $task->setTaskType($taskType); + $task->setAppId($appId); + $task->setStatus($status); + $task->setCategory($category); + $task->setIdentifier($identifier); + return $this->insert($task); + } + + /** + * @param string $userId + * @return void + * @throws Exception + */ + public function deleteUserMetaTasks(string $userId): void { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where( + $qb->expr()->eq('user_id', $qb->createNamedParameter($userId, IQueryBuilder::PARAM_STR)) + ); + $qb->executeStatement(); + $qb->resetQueryParts(); + } + + /** + * Clean up tasks older than specified seconds + * + * @param int $olderThanSeconds + * @return int number of deleted rows + * @throws Exception + * @throws \RuntimeException + */ + public function cleanupOldMetaTasks(int $olderThanSeconds = Application::DEFAULT_ASSISTANT_TASK_IDLE_TIME): int { + $qb = $this->db->getQueryBuilder(); + $qb->delete($this->getTableName()) + ->where( + $qb->expr()->lt('timestamp', $qb->createNamedParameter((new DateTime())->getTimestamp() - $olderThanSeconds, IQueryBuilder::PARAM_INT)) + ); + return $qb->executeStatement(); + } +} diff --git a/lib/Db/SpeechToText/Transcript.php b/lib/Db/SpeechToText/Transcript.php deleted file mode 100644 index 8d52a317..00000000 --- a/lib/Db/SpeechToText/Transcript.php +++ /dev/null @@ -1,69 +0,0 @@ - - * - * @author Anupam Kumar - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -namespace OCA\TpAssistant\Db\SpeechToText; - -use OCP\AppFramework\Db\Entity; -use OCP\DB\Types; - -/** - * Class Transcript - * - * @package OCA\Stt\Db - * @method ?string getUserId() - * @method void setUserId(?string $userId) - * @method string getTranscript() - * @method void setTranscript(string $transcript) - * @method \DateTime getLastAccessed() - * @method void setLastAccessed(\DateTime $lastAccessed) - */ -class Transcript extends Entity { - - protected $userId; - protected $transcript; - protected $lastAccessed; - - public static $columns = [ - 'id', - 'user_id', - 'transcript', - 'last_accessed', - ]; - public static $fields = [ - 'id', - 'userId', - 'transcript', - 'lastAccessed', - ]; - - public function __construct() { - $this->addType('id', Types::INTEGER); - $this->addType('userId', Types::STRING); - $this->addType('transcript', Types::STRING); - $this->addType('lastAccessed', Types::DATETIME); - } -} diff --git a/lib/Db/SpeechToText/TranscriptMapper.php b/lib/Db/SpeechToText/TranscriptMapper.php deleted file mode 100644 index 53691c55..00000000 --- a/lib/Db/SpeechToText/TranscriptMapper.php +++ /dev/null @@ -1,91 +0,0 @@ - - * - * @author Anupam Kumar - * - * @license GNU AGPL version 3 or any later version - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -namespace OCA\TpAssistant\Db\SpeechToText; - -use DateTime; -use Exception; -use OCP\AppFramework\Db\DoesNotExistException; -use OCP\AppFramework\Db\MultipleObjectsReturnedException; -use OCP\AppFramework\Db\QBMapper; -use OCP\DB\QueryBuilder\IQueryBuilder; -use OCP\IDBConnection; -use Psr\Log\LoggerInterface; - -/** - * @template-extends QBMapper - */ -class TranscriptMapper extends QBMapper { - - public function __construct(IDBConnection $db, private LoggerInterface $logger) { - parent::__construct($db, 'assistant_stt_transcripts', Transcript::class); - $this->db = $db; - } - - /** - * @param integer $id - * @param string|null $userId - * @throws Exception - * @throws MultipleObjectsReturnedException if more than one item exist - * @throws DoesNotExistException if the item does not exist - * @return Transcript - */ - public function find(int $id, ?string $userId): Transcript { - $qb = $this->db->getQueryBuilder(); - - if (strlen($userId) > 0 && $userId !== 'admin') { - $qb - ->select(Transcript::$columns) - ->from($this->getTableName()) - ->where($qb->expr()->eq('id', $qb->createNamedParameter($id))) - ->andWhere($qb->expr()->eq('user_id', $qb->createNamedParameter($userId))) - ; - } else { - $qb - ->select(Transcript::$columns) - ->from($this->getTableName()) - ->where($qb->expr()->eq('id', $qb->createNamedParameter($id))) - ; - } - - return $this->findEntity($qb); - } - - public function cleanupTranscriptions(): void { - $qb = $this->db->getQueryBuilder(); - $qb - ->delete($this->getTableName()) - ->where($qb->expr()->lte( - 'last_accessed', - $qb->createNamedParameter(new DateTime('-2 weeks'), IQueryBuilder::PARAM_DATE) - )) - ; - - $deletedRows = $qb->executeStatement(); - $this->logger->debug('Cleared {count} old transcriptions', ['count' => $deletedRows]); - } -} diff --git a/lib/Db/Text2Image/ImageGeneration.php b/lib/Db/Text2Image/ImageGeneration.php index 59e06707..d0586aa3 100644 --- a/lib/Db/Text2Image/ImageGeneration.php +++ b/lib/Db/Text2Image/ImageGeneration.php @@ -9,16 +9,16 @@ use OCP\AppFramework\Db\Entity; /** - * @method string getImageGenId() - * @method void setImageGenId(string $imageGenId) - * @method string getPrompt() - * @method void setPrompt(string $prompt) - * @method void setUserId(string $userId) - * @method string getUserId() - * @method void setTimestamp(int $timestamp) - * @method int getTimestamp() - * @method void setExpGenTime(int $expGenTime) - * @method int getExpGenTime() + * @method \string getImageGenId() + * @method \void setImageGenId(string $imageGenId) + * @method \string getPrompt() + * @method \void setPrompt(string $prompt) + * @method \void setUserId(string $userId) + * @method \string getUserId() + * @method \void setTimestamp(int $timestamp) + * @method \int getTimestamp() + * @method \void setExpGenTime(int $expGenTime) + * @method \int getExpGenTime() * */ class ImageGeneration extends Entity implements \JsonSerializable { diff --git a/lib/Db/Text2Image/ImageGenerationMapper.php b/lib/Db/Text2Image/ImageGenerationMapper.php index 477a508a..eda48ba4 100644 --- a/lib/Db/Text2Image/ImageGenerationMapper.php +++ b/lib/Db/Text2Image/ImageGenerationMapper.php @@ -9,7 +9,6 @@ use DateTime; use OCA\TpAssistant\AppInfo\Application; use OCP\AppFramework\Db\DoesNotExistException; -use OCP\AppFramework\Db\Entity; use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\AppFramework\Db\QBMapper; use OCP\DB\Exception; @@ -30,12 +29,12 @@ public function __construct( /** * @param string $imageGenId - * @return ImageGeneration|Entity + * @return ImageGeneration * @throws DoesNotExistException * @throws Exception * @throws MultipleObjectsReturnedException */ - public function getImageGenerationOfImageGenId(string $imageGenId): ImageGeneration|Entity { + public function getImageGenerationOfImageGenId(string $imageGenId): ImageGeneration { $qb = $this->db->getQueryBuilder(); $qb->select('*') @@ -52,10 +51,10 @@ public function getImageGenerationOfImageGenId(string $imageGenId): ImageGenerat * @param string $prompt * @param string $userId * @param int|null $expCompletionTime - * @return ImageGeneration|Entity + * @return ImageGeneration * @throws Exception */ - public function createImageGeneration(string $imageGenId, string $prompt = '', string $userId = '', ?int $expCompletionTime = null): ImageGeneration|Entity { + public function createImageGeneration(string $imageGenId, string $prompt = '', string $userId = '', ?int $expCompletionTime = null): ImageGeneration { $imageGeneration = new ImageGeneration(); $imageGeneration->setImageGenId($imageGenId); $imageGeneration->setTimestamp((new DateTime())->getTimestamp()); @@ -171,7 +170,7 @@ public function setNotifyReady(string $imageGenId, bool $notifyReady): int { /** * @param int $maxAge - * @return array ('deleted_generations' => int, 'file_names' => string[]) + * @return array{deleted_generations: int, file_names: array} * @throws Exception * @throws \RuntimeException */ @@ -188,11 +187,9 @@ public function cleanupImageGenerations(int $maxAge = Application::DEFAULT_MAX_I $qb->expr()->lt('timestamp', $qb->createNamedParameter($maxTimestamp, IQueryBuilder::PARAM_INT)) ); - /** @var ImageGeneration[] $generations */ $generations = $this->findEntities($qb); $qb->resetQueryParts(); - /** @var array[] $fileNames */ $fileNames = []; $imageGenIds = []; $generationIds = []; diff --git a/lib/Listener/SpeechToText/SpeechToTextResultListener.php b/lib/Listener/SpeechToText/SpeechToTextResultListener.php index f38457d2..8c76e9a5 100644 --- a/lib/Listener/SpeechToText/SpeechToTextResultListener.php +++ b/lib/Listener/SpeechToText/SpeechToTextResultListener.php @@ -23,7 +23,8 @@ namespace OCA\TpAssistant\Listener\SpeechToText; use OCA\TpAssistant\AppInfo\Application; -use OCA\TpAssistant\Service\SpeechToText\SpeechToTextService; +use OCA\TpAssistant\Db\MetaTaskMapper; +use OCA\TpAssistant\Service\AssistantService; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; use OCP\SpeechToText\Events\AbstractTranscriptionEvent; @@ -36,8 +37,9 @@ */ class SpeechToTextResultListener implements IEventListener { public function __construct( - private SpeechToTextService $sttService, - private LoggerInterface $logger, + private LoggerInterface $logger, + private MetaTaskMapper $metaTaskMapper, + private AssistantService $assistantService, ) { } @@ -48,25 +50,69 @@ public function handle(Event $event): void { if ($event instanceof TranscriptionSuccessfulEvent) { $transcript = $event->getTranscript(); - $userId = $event->getUserId(); + $file = $event->getFile(); + + $metaTasks = $this->metaTaskMapper->getMetaTasksByOcpTaskIdAndCategory($file->getId(), Application::TASK_CATEGORY_SPEECH_TO_TEXT); + + // Find a matching etag: + $etag = $file->getEtag(); + $assistantTask = null; + foreach ($metaTasks as $metaTask) { + $metaTaskEtag = $metaTask->getInputsAsArray()['eTag']; + if ($metaTaskEtag === $etag) { + $assistantTask = $metaTask; + break; + } + } + + if ($assistantTask === null) { + $this->logger->error('No assistant task found for speech to text result out of ' . count($metaTasks) . ' tasks for file ' . $file->getId() . ' with etag ' . $etag); + return; + } + + // Update the meta task with the output and new status + $assistantTask->setOutput($transcript); + $assistantTask->setStatus(Application::STT_TASK_SUCCESSFUL); + $assistantTask = $this->metaTaskMapper->update($assistantTask); try { - $this->sttService->sendSpeechToTextNotification($userId, $transcript, true); + $this->assistantService->sendNotification($assistantTask, null, null, $transcript); } catch (\InvalidArgumentException $e) { $this->logger->error('Failed to dispatch notification for successful transcription: ' . $e->getMessage()); } } if ($event instanceof TranscriptionFailedEvent) { - $userId = $event->getUserId(); $this->logger->error('Transcript generation failed: ' . $event->getErrorMessage()); - + + $metaTasks = $this->metaTaskMapper->getMetaTasksByOcpTaskIdAndCategory($file->getId(), Application::TASK_CATEGORY_SPEECH_TO_TEXT); + + // Find a matching etag: + $etag = $file->getEtag(); + $assistantTask = null; + foreach ($metaTasks as $metaTask) { + $metaTaskEtag = $metaTask->getInputsAsArray()['eTag']; + if ($metaTaskEtag === $etag) { + $assistantTask = $metaTask; + break; + } + } + + if ($assistantTask === null) { + $this->logger->error('No assistant task found for speech to text result'); + return; + } + + // Update the meta task with the new status + $assistantTask->setStatus(Application::STT_TASK_FAILED); + $assistantTask = $this->metaTaskMapper->update($assistantTask); + try { - $this->sttService->sendSpeechToTextNotification($userId, '', false); + $this->assistantService->sendNotification($assistantTask); } catch (\InvalidArgumentException $e) { $this->logger->error('Failed to dispatch notification for failed transcription: ' . $e->getMessage()); } - + } } } diff --git a/lib/Listener/TaskFailedListener.php b/lib/Listener/TaskFailedListener.php index 5eb10980..058d76e6 100644 --- a/lib/Listener/TaskFailedListener.php +++ b/lib/Listener/TaskFailedListener.php @@ -3,8 +3,10 @@ namespace OCA\TpAssistant\Listener; use OCA\TpAssistant\AppInfo\Application; +use OCA\TpAssistant\Db\MetaTaskMapper; use OCA\TpAssistant\Event\BeforeAssistantNotificationEvent; use OCA\TpAssistant\Service\AssistantService; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventDispatcher; use OCP\EventDispatcher\IEventListener; @@ -18,6 +20,7 @@ class TaskFailedListener implements IEventListener { public function __construct( private AssistantService $assistantService, private IEventDispatcher $eventDispatcher, + private MetaTaskMapper $metaTaskMapper, ) { } @@ -47,6 +50,18 @@ public function handle(Event $event): void { $notificationActionLabel = $beforeAssistantNotificationEvent->getNotificationActionLabel(); } - $this->assistantService->sendNotification($task, $notificationTarget, $notificationActionLabel); + try { + $assistantTask = $this->metaTaskMapper->getMetaTaskByOcpTaskIdAndCategory($task->getId(), Application::TASK_CATEGORY_TEXT_GEN); + } catch (DoesNotExistException $e) { + // Not an assistant task + return; + } + + // Update task status and output: + $assistantTask->setStatus($task->getStatus()); + $assistantTask->setOutput($task->getOutput()); + $assistantTask = $this->metaTaskMapper->update($assistantTask); + + $this->assistantService->sendNotification($assistantTask, $notificationTarget, $notificationActionLabel); } } diff --git a/lib/Listener/TaskSuccessfulListener.php b/lib/Listener/TaskSuccessfulListener.php index ecbe1203..0bf72bc3 100644 --- a/lib/Listener/TaskSuccessfulListener.php +++ b/lib/Listener/TaskSuccessfulListener.php @@ -3,8 +3,10 @@ namespace OCA\TpAssistant\Listener; use OCA\TpAssistant\AppInfo\Application; +use OCA\TpAssistant\Db\MetaTaskMapper; use OCA\TpAssistant\Event\BeforeAssistantNotificationEvent; use OCA\TpAssistant\Service\AssistantService; +use OCP\AppFramework\Db\DoesNotExistException; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventDispatcher; use OCP\EventDispatcher\IEventListener; @@ -18,6 +20,7 @@ class TaskSuccessfulListener implements IEventListener { public function __construct( private AssistantService $assistantService, private IEventDispatcher $eventDispatcher, + private MetaTaskMapper $metaTaskMapper, ) { } @@ -47,6 +50,18 @@ public function handle(Event $event): void { $notificationActionLabel = $beforeAssistantNotificationEvent->getNotificationActionLabel(); } - $this->assistantService->sendNotification($task, $notificationTarget, $notificationActionLabel); + try { + $assistantTask = $this->metaTaskMapper->getMetaTaskByOcpTaskIdAndCategory($task->getId(), Application::TASK_CATEGORY_TEXT_GEN); + } catch (DoesNotExistException $e) { + // Not an assistant task + return; + } + + // Update task status and output: + $assistantTask->setStatus($task->getStatus()); + $assistantTask->setOutput($task->getOutput()); + $assistantTask = $this->metaTaskMapper->update($assistantTask); + + $this->assistantService->sendNotification($assistantTask, $notificationTarget, $notificationActionLabel); } } diff --git a/lib/Listener/Text2Image/Text2ImageResultListener.php b/lib/Listener/Text2Image/Text2ImageResultListener.php index b831a433..2d42f7ab 100644 --- a/lib/Listener/Text2Image/Text2ImageResultListener.php +++ b/lib/Listener/Text2Image/Text2ImageResultListener.php @@ -3,7 +3,7 @@ namespace OCA\TpAssistant\Listener\Text2Image; use OCA\TpAssistant\AppInfo\Application; -use OCA\TpAssistant\Db\Text2Image\ImageGeneration; +use OCA\TpAssistant\Db\MetaTaskMapper; use OCA\TpAssistant\Db\Text2Image\ImageGenerationMapper; use OCA\TpAssistant\Service\AssistantService; use OCA\TpAssistant\Service\Text2Image\Text2ImageHelperService; @@ -11,11 +11,10 @@ use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\EventDispatcher\Event; use OCP\EventDispatcher\IEventListener; -use OCP\IImage; -use OCP\IURLGenerator; use OCP\TextToImage\Events\AbstractTextToImageEvent; use OCP\TextToImage\Events\TaskFailedEvent; use OCP\TextToImage\Events\TaskSuccessfulEvent; +use OCP\TextToImage\Task; use Psr\Log\LoggerInterface; /** @@ -25,10 +24,10 @@ class Text2ImageResultListener implements IEventListener { public function __construct( private Text2ImageHelperService $text2ImageService, - private ImageGenerationMapper $imageGenerationMapper, - private LoggerInterface $logger, - private AssistantService $assistantService, - private IURLGenerator $urlGenerator, + private ImageGenerationMapper $imageGenerationMapper, + private LoggerInterface $logger, + private AssistantService $assistantService, + private MetaTaskMapper $metaTaskMapper, ) { } @@ -40,7 +39,7 @@ public function handle(Event $event): void { if (!$event instanceof AbstractTextToImageEvent || $event->getTask()->getAppId() !== Application::APP_ID) { return; } - $this->logger->debug("TextToImageEvent received"); + $this->logger->debug('TextToImageEvent received'); $imageGenId = $event->getTask()->getIdentifier(); @@ -49,38 +48,35 @@ public function handle(Event $event): void { return; } - $link = null; // A link to the image generation page (if the task succeeded) + $assistantTask = $this->metaTaskMapper->getMetaTaskByOcpTaskIdAndCategory($event->getTask()->getId(), Application::TASK_CATEGORY_TEXT_TO_IMAGE); if ($event instanceof TaskSuccessfulEvent) { - $this->logger->debug("TextToImageEvent succeeded"); - /** @var IImage $image */ + $this->logger->debug('TextToImageEvent succeeded'); $images = $event->getTask()->getOutputImages(); $this->text2ImageService->storeImages($images, $imageGenId); - // Generate the link for the notification - $link = $this->urlGenerator->linkToRouteAbsolute( - Application::APP_ID . '.Text2Image.showGenerationPage', - [ - 'imageGenId' => $imageGenId, - ] - ); + $assistantTask->setStatus(Task::STATUS_SUCCESSFUL); + $assistantTask = $this->metaTaskMapper->update($assistantTask); } if ($event instanceof TaskFailedEvent) { $this->logger->warning('Image generation task failed: ' . $imageGenId); $this->imageGenerationMapper->setFailed($imageGenId, true); - $this->assistantService->sendNotification($event->getTask()); + // Update the assistant meta task status: + $assistantTask->setStatus(Task::STATUS_FAILED); + $assistantTask = $this->metaTaskMapper->update($assistantTask); + + $this->assistantService->sendNotification($assistantTask); } // Only send the notification if the user enabled them for this task: try { - /** @var ImageGeneration $imageGeneration */ $imageGeneration = $this->imageGenerationMapper->getImageGenerationOfImageGenId($imageGenId); if ($imageGeneration->getNotifyReady()) { - $this->assistantService->sendNotification($event->getTask(), $link); + $this->assistantService->sendNotification($assistantTask); } } catch (\OCP\Db\Exception | DoesNotExistException | MultipleObjectsReturnedException $e) { $this->logger->warning('Could not notify user of a generation (id:' . $imageGenId . ') being ready: ' . $e->getMessage()); diff --git a/lib/Migration/Version010004Date20240131182344.php b/lib/Migration/Version010004Date20240131182344.php new file mode 100644 index 00000000..515ad7c1 --- /dev/null +++ b/lib/Migration/Version010004Date20240131182344.php @@ -0,0 +1,113 @@ + +// SPDX-License-Identifier: AGPL-3.0-or-later + +declare(strict_types=1); + +namespace OCA\TpAssistant\Migration; + +use Closure; +use OCP\DB\ISchemaWrapper; +use OCP\DB\Types; +use OCP\Migration\IOutput; +use OCP\Migration\SimpleMigrationStep; + +class Version010004Date20240131182344 extends SimpleMigrationStep { + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return void + */ + public function preSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void { + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return null|ISchemaWrapper + */ + public function changeSchema(IOutput $output, Closure $schemaClosure, array $options) { + /** @var ISchemaWrapper $schema */ + $schema = $schemaClosure(); + $schemaChanged = false; + + if ($schema->hasTable('assistant_stt_transcripts')) { + // Storing transcripts has been moved to the assistant meta task wrapper + $schemaChanged = true; + $table = $schema->getTable('assistant_stt_transcripts'); + $table->dropIndex('assistant_stt_transcript_user'); + $table->dropIndex('assistant_stt_transcript_la'); + $schema->dropTable('assistant_stt_transcripts'); + } + + if ($schema->hasTable('assistant_text_tasks')) { + $schemaChanged = true; + $table = $schema->getTable('assistant_text_tasks'); + $table->dropIndex('assistant_t_tasks_uid'); + $table->dropIndex('assistant_t_task_id_category'); + $schema->dropTable('assistant_text_tasks'); + } + + if (!$schema->hasTable('assistant_meta_tasks')) { + $schemaChanged = true; + $table = $schema->createTable('assistant_meta_tasks'); + $table->addColumn('id', Types::BIGINT, [ + 'autoincrement' => true, + 'notnull' => true, + ]); + $table->addColumn('user_id', Types::STRING, [ + 'notnull' => true, + 'length' => 64, + ]); + $table->addColumn('app_id', Types::STRING, [ + 'notnull' => true, + 'length' => 32, + ]); + $table->addColumn('inputs', Types::TEXT, [ + 'notnull' => false, + ]); + $table->addColumn('output', Types::TEXT, [ + 'notnull' => false, + ]); + $table->addColumn('ocp_task_id', Types::BIGINT, [ + 'notnull' => false, + ]); + $table->addColumn('timestamp', Types::BIGINT, [ + 'notnull' => true, + 'unsigned' => true, + ]); + $table->addColumn('task_type', Types::STRING, [ + 'notnull' => false, + 'length' => 255, + ]); + $table->addColumn('status', Types::INTEGER, [ + 'notnull' => true, + 'default' => 0, // 0 = Unknown + ]); + $table->addColumn('category', Types::INTEGER, [ + 'notnull' => false, + ]); + $table->addColumn('identifier', Types::STRING, [ + 'notnull' => false, + 'length' => 255, + ]); + $table->setPrimaryKey(['id']); + $table->addIndex(['user_id'], 'assistant_meta_task_uid'); + $table->addIndex(['ocp_task_id','category'], 'assistant_meta_task_id_cat'); + } + + return $schemaChanged ? $schema : null; + } + + /** + * @param IOutput $output + * @param Closure $schemaClosure The `\Closure` returns a `ISchemaWrapper` + * @param array $options + * @return void + */ + public function postSchemaChange(IOutput $output, Closure $schemaClosure, array $options): void { + } +} diff --git a/lib/Notification/Notifier.php b/lib/Notification/Notifier.php index 04006341..856878b4 100644 --- a/lib/Notification/Notifier.php +++ b/lib/Notification/Notifier.php @@ -70,18 +70,25 @@ public function prepare(INotification $notification, string $languageCode): INot $schedulingAppName = $schedulingAppInfo['name']; $taskTypeName = null; - if ($params['taskType'] === Application::TASK_TYPE_TEXT_GEN && - isset($params['taskTypeClass']) && $params['taskTypeClass']) { - try { - /** @var ITaskType $taskType */ - $taskType = $this->container->get($params['taskTypeClass']); - $taskTypeName = $taskType->getName(); - } catch (\Exception | \Throwable $e) { - $this->logger->debug('Impossible to get task type ' . $params['taskTypeClass'], ['exception' => $e]); + $taskInput = $params['inputs']['prompt'] ?? null; + if ($params['taskCategory'] === Application::TASK_CATEGORY_TEXT_GEN) { + + if ($params['taskTypeClass'] === 'copywriter') { + // Catch the custom copywriter task type built on top of the FreePrompt task type. + $taskTypeName = $l->t('Copywriting'); + $taskInput = $l->t('Writing style: %1$s; Source material: %2$s', [$params['inputs']['writingStyle'], $params['inputs']['sourceMaterial']]); + } else { + try { + /** @var ITaskType $taskType */ + $taskType = $this->container->get($params['taskTypeClass']); + $taskTypeName = $taskType->getName(); + } catch (\Exception | \Throwable $e) { + $this->logger->debug('Impossible to get task type ' . $params['taskTypeClass'], ['exception' => $e]); + } } - } elseif ($params['taskType'] === Application::TASK_TYPE_TEXT_TO_IMAGE) { + } elseif ($params['taskCategory'] === Application::TASK_CATEGORY_TEXT_TO_IMAGE) { $taskTypeName = $l->t('Text to image'); - } elseif ($params['taskType'] === Application::TASK_TYPE_SPEECH_TO_TEXT) { + } elseif ($params['taskCategory'] === Application::TASK_CATEGORY_SPEECH_TO_TEXT) { $taskTypeName = $l->t('Speech to text'); } @@ -92,16 +99,17 @@ public function prepare(INotification $notification, string $languageCode): INot : $l->t('"%1$s" task for "%2$s" has finished', [$taskTypeName, $schedulingAppName]); $content = ''; - if (isset($params['input'])) { - $content .= $l->t('Input: %1$s', [$params['input']]); + + if ($taskInput) { + $content .= $l->t('Input: %1$s', [$taskInput]); } - + if (isset($params['result'])) { $content === '' ?: $content .= '\n'; $content .= $l->t('Result: %1$s', [$params['result']]); } - - $link = $params['target'] ?? $this->url->linkToRouteAbsolute(Application::APP_ID . '.assistant.getTextProcessingTaskResultPage', ['taskId' => $params['id']]); + + $link = $params['target']; $iconUrl = $this->url->getAbsoluteURL($this->url->imagePath(Application::APP_ID, 'app-dark.svg')); $notification diff --git a/lib/Service/AssistantService.php b/lib/Service/AssistantService.php index 9a6f5492..35c36309 100644 --- a/lib/Service/AssistantService.php +++ b/lib/Service/AssistantService.php @@ -2,61 +2,102 @@ namespace OCA\TpAssistant\Service; +require_once __DIR__ . '/../../vendor/autoload.php'; + use DateTime; use OCA\TpAssistant\AppInfo\Application; +use OCA\TpAssistant\Db\MetaTask; +use OCA\TpAssistant\Db\MetaTaskMapper; +use OCA\TpAssistant\Db\Text2Image\ImageGenerationMapper; +use OCP\AppFramework\Db\DoesNotExistException; +use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\Common\Exception\NotFoundException; +use OCP\DB\Exception; +use OCP\Files\File; +use OCP\Files\GenericFileException; +use OCP\Files\IRootFolder; +use OCP\Files\NotPermittedException; use OCP\IURLGenerator; +use OCP\Lock\LockedException; use OCP\Notification\IManager as INotificationManager; use OCP\PreConditionNotMetException; +use OCP\TextProcessing\FreePromptTaskType; use OCP\TextProcessing\IManager as ITextProcessingManager; use OCP\TextProcessing\Task as TextProcessingTask; use OCP\TextToImage\Task as TextToImageTask; +use Parsedown; +use PhpOffice\PhpWord\IOFactory; +use Psr\Log\LoggerInterface; +use RuntimeException; class AssistantService { public function __construct( - private INotificationManager $notificationManager, + private INotificationManager $notificationManager, private ITextProcessingManager $textProcessingManager, - private IURLGenerator $url, + private MetaTaskMapper $metaTaskMapper, + private ImageGenerationMapper $imageGenerationMapper, + private LoggerInterface $logger, + private IRootFolder $storage, + private IURLGenerator $url, ) { } /** * Send a success or failure task result notification * - * @param TextProcessingTask|TextToImageTask $task - * @param string|null $target optional notification link target + * @param MetaTask $task + * @param string|null $customTarget optional notification link target * @param string|null $actionLabel optional label for the notification action button + * @param string|null $resultPreview * @return void - * @throws \InvalidArgumentException */ - public function sendNotification(TextProcessingTask|TextToImageTask $task, ?string $target = null, ?string $actionLabel = null): void { + public function sendNotification(MetaTask $task, ?string $customTarget = null, ?string $actionLabel = null, ?string $resultPreview = null): void { $manager = $this->notificationManager; $notification = $manager->createNotification(); $params = [ 'appId' => $task->getAppId(), 'id' => $task->getId(), - 'input' => $task->getInput(), - 'target' => $target, + 'inputs' => $task->getInputsAsArray(), + 'target' => $customTarget ?? $this->getDefaultTarget($task), 'actionLabel' => $actionLabel, + 'result' => $resultPreview, ]; - if ($task instanceof TextToImageTask) { - $params['taskType'] = Application::TASK_TYPE_TEXT_TO_IMAGE; - $subject = $task->getStatus() === TextToImageTask::STATUS_SUCCESSFUL - ? 'success' - : 'failure'; - } else { - $params['taskType'] = Application::TASK_TYPE_TEXT_GEN; - $params['textTaskTypeClass'] = $task->getType(); - $subject = $task->getStatus() === TextProcessingTask::STATUS_SUCCESSFUL - ? 'success' - : 'failure'; + $params['taskTypeClass'] = $task->getTaskType(); + $params['taskCategory'] = $task->getCategory(); + + switch ($task->getCategory()) { + case Application::TASK_CATEGORY_TEXT_TO_IMAGE: + { + $taskSuccessful = $task->getStatus() === TextToImageTask::STATUS_SUCCESSFUL; + break; + } + case Application::TASK_CATEGORY_TEXT_GEN: + { + $taskSuccessful = $task->getStatus() === TextProcessingTask::STATUS_SUCCESSFUL; + break; + } + case Application::TASK_CATEGORY_SPEECH_TO_TEXT: + { + $taskSuccessful = $task->getStatus() === Application::STT_TASK_SUCCESSFUL; + break; + } + default: + { + $taskSuccessful = false; + break; + } } - $objectType = $target === null + $subject = $taskSuccessful + ? 'success' + : 'failure'; + + $objectType = $customTarget === null ? 'task' : 'task-with-custom-target'; + $notification->setApp(Application::APP_ID) ->setUser($task->getUserId()) ->setDateTime(new DateTime()) @@ -66,50 +107,331 @@ public function sendNotification(TextProcessingTask|TextToImageTask $task, ?stri $manager->notify($notification); } + private function getDefaultTarget(MetaTask $task): string { + $category = $task->getCategory(); + if ($category === Application::TASK_CATEGORY_TEXT_GEN) { + return $this->url->linkToRouteAbsolute(Application::APP_ID . '.assistant.getTextProcessingTaskResultPage', ['taskId' => $task->getId()]); + } elseif ($category === Application::TASK_CATEGORY_SPEECH_TO_TEXT) { + return $this->url->linkToRouteAbsolute(Application::APP_ID . '.SpeechToText.getResultPage', ['id' => $task->getId()]); + } elseif ($category === Application::TASK_CATEGORY_TEXT_TO_IMAGE) { + $imageGeneration = $this->imageGenerationMapper->getImageGenerationOfImageGenId($task->getIdentifier()); + return $this->url->linkToRouteAbsolute( + Application::APP_ID . '.Text2Image.showGenerationPage', + [ + 'imageGenId' => $imageGeneration->getImageGenId(), + ] + ); + } + return ''; + } + /** - * @param string|null $userId + * @param string $writingStyle + * @param string $sourceMaterial + * @return string + */ + private function formattedCopywriterPrompt(string $writingStyle, string $sourceMaterial): string { + return "You're a professional copywriter tasked with copying an instructed or demonstrated *WRITING STYLE* and writing a text on the provided *SOURCE MATERIAL*. \n*WRITING STYLE*:\n$writingStyle\n\n*SOURCE MATERIAL*:\n\n$sourceMaterial\n\nNow write a text in the same style detailed or demonstrated under *WRITING STYLE* using the *SOURCE MATERIAL* as source of facts and instruction on what to write about. Do not invent any facts or events yourself. Also, use the *WRITING STYLE* as a guide for how to write the text ONLY and not as a source of facts or events."; + } + + /** + * Sanitize inputs for storage based on the input type + * @param string $type + * @param array $inputs + * @return array + * @throws \Exception + */ + private function sanitizeInputs(string $type, array $inputs): array { + switch ($type) { + case 'copywriter': + { + // Sanitize the input array based on the allowed keys and making sure all inputs are strings: + $inputs = array_filter($inputs, function ($value, $key) { + return in_array($key, ['writingStyle', 'sourceMaterial']) && is_string($value); + }, ARRAY_FILTER_USE_BOTH); + + if (count($inputs) !== 2) { + throw new \Exception('Invalid input(s)'); + } + break; + } + default: + { + if (!is_string($inputs['prompt']) || count($inputs) !== 1) { + throw new \Exception('Invalid input(s)'); + } + break; + } + } + return $inputs; + } + + /** + * @param string $userId * @param int $taskId - * @return TextProcessingTask + * @return MetaTask|null */ - public function getTextProcessingTask(?string $userId, int $taskId): ?TextProcessingTask { + public function getTextProcessingTask(string $userId, int $taskId): ?MetaTask { try { - $task = $this->textProcessingManager->getTask($taskId); - } catch (NotFoundException | \RuntimeException $e) { + $metaTask = $this->metaTaskMapper->getMetaTask($taskId); + } catch (DoesNotExistException | MultipleObjectsReturnedException | \OCP\Db\Exception $e) { return null; } - if ($task->getUserId() !== $userId) { + if ($metaTask->getUserId() !== $userId) { return null; } - return $task; + // Check if the task status is up-to-date (if not, update status and output) + try { + $ocpTask = $this->textProcessingManager->getTask($metaTask->getOcpTaskId()); + + if($ocpTask->getStatus() !== $metaTask->getStatus()) { + $metaTask->setStatus($ocpTask->getStatus()); + $metaTask->setOutput($ocpTask->getOutput()); + $metaTask = $this->metaTaskMapper->update($metaTask); + } + } catch (NotFoundException $e) { + // Ocp task not found, so we can't update the status + $this->logger->debug('OCP task not found for assistant task ' . $metaTask->getId() . '. Could not update status.'); + } catch (\InvalidArgumentException | \OCP\Db\Exception | RuntimeException $e) { + // Something else went wrong, so we can't update the status + $this->logger->warning('Unknown error while trying to retreive an updated status for assistant task: ' . $metaTask->getId() . '.', ['exception' => $e]); + } + + return $metaTask; } /** * @param string $type - * @param string $input + * @param array $inputs * @param string $appId - * @param string|null $userId + * @param string $userId * @param string $identifier - * @return TextProcessingTask + * @return MetaTask * @throws PreConditionNotMetException + * @throws Exception */ - public function runTextProcessingTask(string $type, string $input, string $appId, ?string $userId, string $identifier): TextProcessingTask { - $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); - $this->textProcessingManager->runTask($task); - return $task; + public function runTextProcessingTask(string $type, array $inputs, string $appId, string $userId, string $identifier): MetaTask { + $inputs = $this->sanitizeInputs($type, $inputs); + switch ($type) { + case 'copywriter': + { + // Format the input prompt + $input = $this->formattedCopywriterPrompt($inputs['writingStyle'], $inputs['sourceMaterial']); + $task = new TextProcessingTask(FreePromptTaskType::class, $input, $appId, $userId, $identifier); + $this->textProcessingManager->runTask($task); + break; + } + default: + { + $input = $inputs['prompt']; + $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); + $this->textProcessingManager->runTask($task); + break; + } + } + + return $this->metaTaskMapper->createMetaTask( + $userId, $inputs, $task->getOutput(), time(), $task->getId(), $type, + $appId, $task->getStatus(), Application::TASK_CATEGORY_TEXT_GEN, $identifier + ); } /** * @param string $type - * @param string $input + * @param array $inputs * @param string $appId - * @param string|null $userId + * @param string $userId * @param string $identifier - * @return TextProcessingTask + * @return MetaTask + * @throws Exception * @throws PreConditionNotMetException */ - public function runOrScheduleTextProcessingTask(string $type, string $input, string $appId, ?string $userId, string $identifier): TextProcessingTask { - $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); - $this->textProcessingManager->runOrScheduleTask($task); - return $task; + public function scheduleTextProcessingTask(string $type, array $inputs, string $appId, string $userId, string $identifier): MetaTask { + $inputs = $this->sanitizeInputs($type, $inputs); + switch ($type) { + case 'copywriter': + { + // Format the input prompt + $input = $this->formattedCopywriterPrompt($inputs['writingStyle'], $inputs['sourceMaterial']); + $task = new TextProcessingTask(FreePromptTaskType::class, $input, $appId, $userId, $identifier); + $this->textProcessingManager->scheduleTask($task); + break; + } + default: + { + $input = $inputs['prompt']; + $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); + $this->textProcessingManager->scheduleTask($task); + break; + } + } + + return $this->metaTaskMapper->createMetaTask( + $userId, $inputs, $task->getOutput(), time(), $task->getId(), $type, + $appId, $task->getStatus(), Application::TASK_CATEGORY_TEXT_GEN, $identifier + ); + } + + /** + * @param string $type + * @param array $inputs + * @param string $appId + * @param string $userId + * @param string $identifier + * @return MetaTask + * @throws PreConditionNotMetException + * @throws \OCP\Db\Exception + * @throws \Exception + */ + public function runOrScheduleTextProcessingTask(string $type, array $inputs, string $appId, string $userId, string $identifier): MetaTask { + $inputs = $this->sanitizeInputs($type, $inputs); + switch ($type) { + case 'copywriter': + { + // Format the input prompt + $input = $this->formattedCopywriterPrompt($inputs['writingStyle'], $inputs['sourceMaterial']); + $task = new TextProcessingTask(FreePromptTaskType::class, $input, $appId, $userId, $identifier); + $this->textProcessingManager->runOrScheduleTask($task); + break; + } + default: + { + $input = $inputs['prompt']; + $task = new TextProcessingTask($type, $input, $appId, $userId, $identifier); + $this->textProcessingManager->runOrScheduleTask($task); + break; + } + } + + return $this->metaTaskMapper->createMetaTask( + $userId, $inputs, $task->getOutput(), time(), $task->getId(), $type, + $appId, $task->getStatus(), Application::TASK_CATEGORY_TEXT_GEN, $identifier + ); + } + + /** + * Parse text from file (if parsing the file type is supported) + * @param string $filePath + * @param string $userId + * @return string + * @throws \Exception + */ + public function parseTextFromFile(string $filePath, string $userId): string { + + try { + $userFolder = $this->storage->getUserFolder($userId); + } catch (\OC\User\NoUserException | NotPermittedException $e) { + throw new \Exception('Could not access user storage.'); + } + + try { + $file = $userFolder->get($filePath); + } catch (NotFoundException $e) { + throw new \Exception('File not found.'); + } + + try { + if ($file instanceof File) { + $contents = $file->getContent(); + } else { + throw new \Exception('Provided path does not point to a file.'); + } + } catch (LockedException | GenericFileException | NotPermittedException $e) { + throw new \Exception('File could not be accessed.'); + } + + $mimeType = $file->getMimeType(); + + switch ($mimeType) { + default: + case 'text/plain': + { + $text = $contents; + + break; + } + case 'text/markdown': + { + $parser = new Parsedown(); + $text = $parser->text($contents); + // Remove HTML tags: + $text = strip_tags($text); + break; + } + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + case 'application/msword': + case 'application/rtf': + case 'application/vnd.oasis.opendocument.text': + { + // Store the file in a temp dir and provide a path for the doc parser to use + $tempFilePath = sys_get_temp_dir() . '/assistant_app/' . uniqid() . '.tmp'; + // Make sure the temp dir exists + if (!file_exists(dirname($tempFilePath))) { + mkdir(dirname($tempFilePath), 0600, true); + } + file_put_contents($tempFilePath, $contents); + + $text = $this->parseDocument($tempFilePath, $mimeType); + + // Remove the hardlink to the file (delete it): + unlink($tempFilePath); + + break; + } + } + return $text; + } + + /** + * Parse text from doc/docx/odt/rtf file + * @param string $filePath + * @param string $mimeType + * @return string + * @throws \Exception + */ + private function parseDocument(string $filePath, string $mimeType): string { + switch ($mimeType) { + case 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': + { + $readerType = 'Word2007'; + break; + } + case 'application/msword': + { + $readerType = 'MsDoc'; + break; + } + case 'application/rtf': + { + $readerType = 'RTF'; + break; + } + case 'application/vnd.oasis.opendocument.text': + { + $readerType = 'ODText'; + break; + } + default: + { + throw new \Exception('Unsupported file mimetype'); + } + } + + + $phpWord = IOFactory::createReader($readerType); + $phpWord = $phpWord->load($filePath); + $sections = $phpWord->getSections(); + $outText = ''; + foreach ($sections as $section) { + $elements = $section->getElements(); + foreach ($elements as $element) { + $class = get_class($element); + if (method_exists($element, 'getText')) { + $outText .= $element->getText() . "\n"; + } + } + } + + return $outText; } } diff --git a/lib/Service/FreePrompt/FreePromptService.php b/lib/Service/FreePrompt/FreePromptService.php index 99d9e90d..2f7e97aa 100644 --- a/lib/Service/FreePrompt/FreePromptService.php +++ b/lib/Service/FreePrompt/FreePromptService.php @@ -8,9 +8,9 @@ use Exception; use OCA\TpAssistant\AppInfo\Application; use OCA\TpAssistant\Db\FreePrompt\PromptMapper; +use OCA\TpAssistant\Db\MetaTaskMapper; use OCP\AppFramework\Http; use OCP\DB\Exception as DBException; -use OCP\IConfig; use OCP\IL10N; use OCP\PreConditionNotMetException; use OCP\TextProcessing\Exception\TaskFailureException; @@ -21,100 +21,99 @@ class FreePromptService { public function __construct( - private IConfig $config, private LoggerInterface $logger, - private IManager $textProcessingManager, - private ?string $userId, - private PromptMapper $promptMapper, - private IL10N $l10n + private IManager $textProcessingManager, + private PromptMapper $promptMapper, + private IL10N $l10n, + private MetaTaskMapper $metaTaskMapper, ) { } /* * @param string $prompt - * @param int $nResults - * @param bool $showPrompt + * @param string $userId * @return string * @throws Exception */ - public function processPrompt(string $prompt, int $nResults): string { + public function processPrompt(string $prompt, string $userId): string { $taskTypes = $this->textProcessingManager->getAvailableTaskTypes(); if (!in_array(FreePromptTaskType::class, $taskTypes)) { $this->logger->warning('FreePromptTaskType not available'); throw new Exception($this->l10n->t('FreePromptTaskType not available'), Http::STATUS_INTERNAL_SERVER_ERROR); } - if ($this->userId === null) { - $this->logger->warning('User id is null when trying to process prompt'); - throw new Exception($this->l10n->t('Failed process prompt; unknown user'), Http::STATUS_INTERNAL_SERVER_ERROR); - } - // Generate a unique id for this generation while (true) { $genId = bin2hex(random_bytes(32)); // Exceedingly unlikely that this will ever happen, but just in case: - if(count($this->textProcessingManager->getUserTasksByApp($this->userId, Application::APP_ID, $genId)) === 0) { + if(count($this->textProcessingManager->getUserTasksByApp($userId, Application::APP_ID, $genId)) === 0) { break; } else { continue; } } - // Generate nResults prompts - for ($i = 0; $i < $nResults; $i++) { - - // Create a db entity for the generation - $promptTask = new Task(FreePromptTaskType::class, $prompt, Application::APP_ID, $this->userId, $genId); - - // Run or schedule the task: - try { - $this->textProcessingManager->runOrScheduleTask($promptTask); - } catch (DBException | PreConditionNotMetException | TaskFailureException $e) { - $this->logger->warning('Failed to run or schedule a task', ['exception' => $e]); - throw new Exception($this->l10n->t('Failed to run or schedule a task'), Http::STATUS_INTERNAL_SERVER_ERROR); - } + $promptTask = new Task(FreePromptTaskType::class, $prompt, Application::APP_ID, $userId, $genId); - // If the task was run immediately, we'll skip the notification.. - // Otherwise we would have to dispatch the notification here. + // Run or schedule the task: + try { + $this->textProcessingManager->runOrScheduleTask($promptTask); + } catch (DBException | PreConditionNotMetException | TaskFailureException $e) { + $this->logger->warning('Failed to run or schedule a task', ['exception' => $e]); + throw new Exception($this->l10n->t('Failed to run or schedule a task'), Http::STATUS_INTERNAL_SERVER_ERROR); } + // Create an assistant task for the free prompt task: + $this->metaTaskMapper->createMetaTask( + $userId, + ['prompt' => $prompt], + $promptTask->getOutput(), + time(), + $promptTask->getId(), + FreePromptTaskType::class, + Application::APP_ID, + $promptTask->getStatus(), + Application::TASK_CATEGORY_TEXT_GEN, + $genId + ); + + // If the task was run immediately, we'll skip the notification.. + // Otherwise we would have to dispatch the notification here. + // Save prompt to database - $this->promptMapper->createPrompt($this->userId, $prompt); - + $this->promptMapper->createPrompt($userId, $prompt); + return $genId; } /** + * @param string $userId * @return array * @throws Exception */ - public function getPromptHistory(): array { - if ($this->userId === null) { - $this->logger->warning('User id is null when trying to get prompt history'); - throw new Exception($this->l10n->t('Failed to get prompt history'), Http::STATUS_INTERNAL_SERVER_ERROR); - } - + public function getPromptHistory(string $userId): array { try { - return $this->promptMapper->getPromptsOfUser($this->userId); + return $this->promptMapper->getPromptsOfUser($userId); } catch (DBException $e) { $this->logger->warning('Failed to get prompts of user', ['exception' => $e]); throw new Exception($this->l10n->t('Failed to get prompt history'), Http::STATUS_INTERNAL_SERVER_ERROR); } } - + /** * @param string $genId + * @param string $userId * @return array * @throws Exception */ - public function getOutputs(string $genId): array { - $tasks = $this->textProcessingManager->getUserTasksByApp($this->userId, Application::APP_ID, $genId); - + public function getOutputs(string $genId, string $userId): array { + $tasks = $this->textProcessingManager->getUserTasksByApp($userId, Application::APP_ID, $genId); + if (count($tasks) === 0) { $this->logger->warning('No tasks found for gen id: ' . $genId); throw new Exception($this->l10n->t('Generation not found'), Http::STATUS_BAD_REQUEST); } - + $outputs = []; /** @var Task $task */ foreach ($tasks as $task) { @@ -130,18 +129,14 @@ public function getOutputs(string $genId): array { /** * @param string $genId + * @param string $userId * @return void * @throws Exception */ - public function cancelGeneration(string $genId): void { - if ($this->userId === null) { - $this->logger->warning('User id is null when trying to cancel generation'); - throw new Exception($this->l10n->t('Failed to cancel generation'), Http::STATUS_INTERNAL_SERVER_ERROR); - } - + public function cancelGeneration(string $genId, string $userId): void { // Get all tasks that have this genId as identifier. /** @var Task[] $tasks */ - $tasks = $this->textProcessingManager->getUserTasksByApp($this->userId, Application::APP_ID, $genId); + $tasks = $this->textProcessingManager->getUserTasksByApp($userId, Application::APP_ID, $genId); // Cancel all tasks foreach ($tasks as $task) { diff --git a/lib/Service/SpeechToText/SpeechToTextService.php b/lib/Service/SpeechToText/SpeechToTextService.php index a810b6bb..67215331 100644 --- a/lib/Service/SpeechToText/SpeechToTextService.php +++ b/lib/Service/SpeechToText/SpeechToTextService.php @@ -25,31 +25,24 @@ use DateTime; use InvalidArgumentException; use OCA\TpAssistant\AppInfo\Application; -use OCA\TpAssistant\Db\SpeechToText\Transcript; -use OCA\TpAssistant\Db\SpeechToText\TranscriptMapper; +use OCA\TpAssistant\Db\MetaTaskMapper; use OCP\Files\File; use OCP\Files\Folder; use OCP\Files\IRootFolder; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\IConfig; -use OCP\IURLGenerator; -use OCP\Notification\IManager as INotifyManager; use OCP\PreConditionNotMetException; use OCP\SpeechToText\ISpeechToTextManager; -use Psr\Log\LoggerInterface; use RuntimeException; class SpeechToTextService { public function __construct( private ISpeechToTextManager $manager, - private IRootFolder $rootFolder, - private INotifyManager $notificationManager, - private IURLGenerator $urlGenerator, - private LoggerInterface $logger, - private IConfig $config, - private TranscriptMapper $transcriptMapper, + private IRootFolder $rootFolder, + private IConfig $config, + private MetaTaskMapper $metaTaskMapper, ) { } @@ -72,6 +65,17 @@ public function transcribeFile(string $path, ?string $userId): void { $audioFile = $userFolder->get($path); $this->manager->scheduleFileTranscription($audioFile, $userId, Application::APP_ID); + + $this->metaTaskMapper->createMetaTask( + $userId, + ['fileId' => $audioFile->getId(), 'eTag' => $audioFile->getEtag()], + '', + time(), + $audioFile->getId(), + "Speech-to-text task", + Application::APP_ID, + Application::STT_TASK_SCHEDULED, + Application::TASK_CATEGORY_SPEECH_TO_TEXT); } /** @@ -88,7 +92,19 @@ public function transcribeAudio(string $tempFileLocation, ?string $userId): void } $audioFile = $this->getFileObject($userId, $tempFileLocation); + $this->manager->scheduleFileTranscription($audioFile, $userId, Application::APP_ID); + + $this->metaTaskMapper->createMetaTask( + $userId, + ['fileId' => $audioFile->getId(), 'eTag' => $audioFile->getEtag()], + '', + time(), + $audioFile->getId(), + "Speech-to-text task", + Application::APP_ID, + Application::STT_TASK_SCHEDULED, + Application::TASK_CATEGORY_SPEECH_TO_TEXT); } /** @@ -150,51 +166,4 @@ private function getUniqueNamedFolder(string $userId, int $try = 3): Folder { return $userFolder->newFolder($sttFolderPath); } - - /** - * Send transcription result notification - * @param string $userId - * @param string $result - * @param boolean $success - * @param int $taskType - * @return void - * @throws \InvalidArgumentException - */ - public function sendSpeechToTextNotification(string $userId, string $result, bool $success): void { - $manager = $this->notificationManager; - $notification = $manager->createNotification(); - - try { - $transcriptEntity = new Transcript(); - $transcriptEntity->setUserId($userId); - $transcriptEntity->setTranscript($result); - // never seen transcripts should also be deleted in the cleanup job - $transcriptEntity->setLastAccessed(new DateTime()); - $transcriptEntity = $this->transcriptMapper->insert($transcriptEntity); - - $id = $transcriptEntity->getId(); - } catch (\OCP\Db\Exception $e) { - $this->logger->error('Failed to save transcript in DB: ' . $e->getMessage()); - $success = false; - $id = 0; - } - - $params = [ - 'appId' => Application::APP_ID, - 'taskType' => Application::TASK_TYPE_SPEECH_TO_TEXT, - 'result' => $result, - 'target' => $this->urlGenerator->linkToRouteAbsolute(Application::APP_ID . '.SpeechToText.getResultPage', ['id' => $id]) - ]; - $subject = $success - ? 'success' - : 'failure'; - - $notification->setApp(Application::APP_ID) - ->setUser($userId) - ->setDateTime(new DateTime()) - ->setObject('speech-to-text-result', (string) $id) - ->setSubject($subject, $params); - - $manager->notify($notification); - } } diff --git a/lib/Service/Text2Image/CleanUpService.php b/lib/Service/Text2Image/CleanUpService.php index c54b1d63..1a47076a 100644 --- a/lib/Service/Text2Image/CleanUpService.php +++ b/lib/Service/Text2Image/CleanUpService.php @@ -8,30 +8,24 @@ use Exception; use OCA\TpAssistant\AppInfo\Application; use OCA\TpAssistant\Db\Text2Image\ImageGenerationMapper; -use OCP\Files\IAppData; use OCP\Files\NotFoundException; use OCP\Files\NotPermittedException; use OCP\IConfig; use Psr\Log\LoggerInterface; use RuntimeException; -/** - * Service to make requests to OpenAI REST API - */ class CleanUpService { public function __construct( private LoggerInterface $logger, private ImageGenerationMapper $imageGenerationMapper, private Text2ImageHelperService $text2ImageHelperService, - private IAppData $appData, private IConfig $config ) { - } /** * @param int|null $maxAge - * @return array ('deleted_files' => int, 'file_deletion_errors' => int, 'deleted_generations' => int) + * @return array{deleted_files: int, file_deletion_errors: int, deleted_generations: int} * @throws Exception */ public function cleanupGenerationsAndFiles(?int $maxAge = null): array { @@ -56,11 +50,9 @@ public function cleanupGenerationsAndFiles(?int $maxAge = null): array { throw new Exception('Image data folder could not be accessed'); } - $deletedFiles = 0; $deletionErrors = 0; - /** @var string $fileName */ foreach ($cleanedUp['file_names'] as $fileName) { try { $imageDataFolder->getFile($fileName)->delete(); @@ -75,8 +67,5 @@ public function cleanupGenerationsAndFiles(?int $maxAge = null): array { ' idle generations. Failed to delete ' . $deletionErrors . ' files.'); return ['deleted_files' => $deletedFiles, 'file_deletion_errors' => $deletionErrors, 'deleted_generations' => $cleanedUp['deleted_generations']]; - } - - } diff --git a/lib/Service/Text2Image/Text2ImageHelperService.php b/lib/Service/Text2Image/Text2ImageHelperService.php index 17118227..eb7840e2 100644 --- a/lib/Service/Text2Image/Text2ImageHelperService.php +++ b/lib/Service/Text2Image/Text2ImageHelperService.php @@ -10,14 +10,15 @@ use GdImage; use OCA\TpAssistant\AppInfo\Application; +use OCA\TpAssistant\Db\MetaTaskMapper; use OCA\TpAssistant\Db\Text2Image\ImageFileName; use OCA\TpAssistant\Db\Text2Image\ImageFileNameMapper; use OCA\TpAssistant\Db\Text2Image\ImageGeneration; use OCA\TpAssistant\Db\Text2Image\ImageGenerationMapper; use OCA\TpAssistant\Db\Text2Image\PromptMapper; use OCA\TpAssistant\Db\Text2Image\StaleGenerationMapper; -use OCA\TpAssistant\Service\AssistantService; +use OCA\TpAssistant\Service\AssistantService; use OCP\AppFramework\Db\DoesNotExistException; use OCP\AppFramework\Db\MultipleObjectsReturnedException; use OCP\AppFramework\Http; @@ -42,17 +43,17 @@ class Text2ImageHelperService { private ?ISimpleFolder $imageDataFolder = null; public function __construct( - private LoggerInterface $logger, - private IManager $textToImageManager, - private ?string $userId, - private PromptMapper $promptMapper, + private LoggerInterface $logger, + private IManager $textToImageManager, + private PromptMapper $promptMapper, private ImageGenerationMapper $imageGenerationMapper, - private ImageFileNameMapper $imageFileNameMapper, + private ImageFileNameMapper $imageFileNameMapper, private StaleGenerationMapper $staleGenerationMapper, - private IAppData $appData, - private IURLGenerator $urlGenerator, - private IL10N $l10n, - private AssistantService $assistantService + private IAppData $appData, + private IURLGenerator $urlGenerator, + private IL10N $l10n, + private AssistantService $assistantService, + private MetaTaskMapper $metaTaskMapper, ) { } @@ -62,13 +63,14 @@ public function __construct( * @param string $prompt * @param int $nResults * @param bool $displayPrompt + * @param string $userId * @return array * @throws Exception * @throws PreConditionNotMetException * @throws TaskFailureException ; * @throws RandomException */ - public function processPrompt(string $prompt, int $nResults, bool $displayPrompt): array { + public function processPrompt(string $prompt, int $nResults, bool $displayPrompt, string $userId): array { if (!$this->textToImageManager->hasProviders()) { $this->logger->error('No text to image processing provider available'); throw new BaseException($this->l10n->t('No text to image processing provider available')); @@ -81,13 +83,12 @@ public function processPrompt(string $prompt, int $nResults, bool $displayPrompt $imageGenId = bin2hex(random_bytes(16)); } - $promptTask = new Task($prompt, Application::APP_ID, $nResults, $this->userId, $imageGenId); + $promptTask = new Task($prompt, Application::APP_ID, $nResults, $userId, $imageGenId); $this->textToImageManager->runOrScheduleTask($promptTask); $taskExecuted = false; - /** @var IImage[]|null $images */ $images = []; $expCompletionTime = new DateTime('now'); @@ -100,7 +101,22 @@ public function processPrompt(string $prompt, int $nResults, bool $displayPrompt } // Store the image id to the db: - $this->imageGenerationMapper->createImageGeneration($imageGenId, $displayPrompt ? $prompt : '', $this->userId ?? '', $expCompletionTime->getTimestamp()); + $this->imageGenerationMapper->createImageGeneration($imageGenId, $displayPrompt ? $prompt : '', $userId, $expCompletionTime->getTimestamp()); + + // Create an assistant meta task for the image generation task: + // TODO check if we should create a task if userId is null + $this->metaTaskMapper->createMetaTask( + $userId, + ['prompt' => $prompt], + $imageGenId, + time(), + $promptTask->getId(), + Task::class, + Application::APP_ID, + $promptTask->getStatus(), + Application::TASK_CATEGORY_TEXT_TO_IMAGE, + $promptTask->getIdentifier() + ); if ($taskExecuted) { $this->storeImages($images, $imageGenId); @@ -121,17 +137,17 @@ public function processPrompt(string $prompt, int $nResults, bool $displayPrompt ); // Save the prompt to database - if($this->userId !== null) { - $this->promptMapper->createPrompt($this->userId, $prompt); - } + $this->promptMapper->createPrompt($userId, $prompt); return ['url' => $infoUrl, 'reference_url' => $referenceUrl, 'image_gen_id' => $imageGenId, 'prompt' => $prompt]; } - /* + /** * Check whether the image generation id exists in the database (stale or otherwise) + * * @param string $imageGenId * @return bool + * @throws BaseException */ private function genIdExists(string $imageGenId): bool { try { @@ -154,22 +170,21 @@ private function genIdExists(string $imageGenId): bool { } /** + * @param string $userId * @return array * @throws \OCP\DB\Exception */ - public function getPromptHistory(): array { - if ($this->userId === null) { - return []; - } else { - return $this->promptMapper->getPromptsOfUser($this->userId); - } + public function getPromptHistory(string $userId): array { + return $this->promptMapper->getPromptsOfUser($userId); } /** * Save image locally as jpg (to save space) + * * @param array|null $iImages * @param string $imageGenId * @return void + * @throws Exception */ public function storeImages(?array $iImages, string $imageGenId): void { if ($iImages === null || count($iImages) === 0) { @@ -233,38 +248,9 @@ public function storeImages(?array $iImages, string $imageGenId): void { // For clarity we'll notify the user that the generation is ready in the event listener } - /** - * Notify user of generation being ready - * @param string $imageGenId - * @return void - */ - public function notifyUser(string $imageGenId): void { - // Get the task associated with the generation: - try { - $task = $this->textToImageManager->getUserTasksByApp(null, Application::APP_ID, $imageGenId); - if (count($task) === 0) { - throw new RuntimeException('empty task array'); - } - } catch (RuntimeException $e) { - $this->logger->debug('Task for the given generation id does not exist or could not be retrieved: ' . $e->getMessage(), ['app' => Application::APP_ID]); - return; - } - - // Generate the link: - $link = $this->urlGenerator->linkToRouteAbsolute( - Application::APP_ID . '.Text2Image.showGenerationPage', - [ - 'imageGenId' => $imageGenId, - ] - ); - - // Notify the user: - $this->assistantService->sendNotification($task[0], $link, $this->l10n->t('View')); - - } - /** * Get imageDataFolder + * * @return ISimpleFolder * @throws \Exception */ @@ -292,14 +278,15 @@ public function getImageDataFolder(): ISimpleFolder { } /** - * Get image generation info. + * Get image generation info + * * @param string $imageGenId + * @param string $userId * @param bool $updateTimestamp - * @param string|null $userId * @return array * @throws \Exception */ - public function getGenerationInfo(string $imageGenId, bool $updateTimestamp = true): array { + public function getGenerationInfo(string $imageGenId, string $userId, bool $updateTimestamp = true): array { // Check whether the task has completed: try { /** @var ImageGeneration $imageGeneration */ @@ -320,7 +307,7 @@ public function getGenerationInfo(string $imageGenId, bool $updateTimestamp = tr throw new BaseException($this->l10n->t('Retrieving the image generation failed.'), Http::STATUS_INTERNAL_SERVER_ERROR); } - $isOwner = ($imageGeneration->getUserId() === $this->userId); + $isOwner = ($imageGeneration->getUserId() === $userId); if ($imageGeneration->getFailed() === true) { throw new BaseException($this->l10n->t('Image generation failed.'), Http::STATUS_INTERNAL_SERVER_ERROR); @@ -368,12 +355,13 @@ public function getGenerationInfo(string $imageGenId, bool $updateTimestamp = tr /** * Get image based on imageFileNameId (imageGenId is used to prevent guessing image ids) + * * @param string $imageGenId * @param int $imageFileNameId - * @return array ('image' => string, 'content-type' => string) + * @return array{image: string, 'content-type': array} * @throws BaseException */ - public function getImage(string $imageGenId, int $imageFileNameId): ?array { + public function getImage(string $imageGenId, int $imageFileNameId): array { try { $generationId = $this->imageGenerationMapper->getImageGenerationOfImageGenId($imageGenId)->getId(); /** @var ImageFileName $imageFileName */ @@ -411,12 +399,14 @@ public function getImage(string $imageGenId, int $imageFileNameId): ?array { /** * Cancel image generation + * * @param string $imageGenId + * @param string $userId * @return void + * @throws NotPermittedException */ - public function cancelGeneration(string $imageGenId): void { + public function cancelGeneration(string $imageGenId, string $userId): void { try { - /** @var ImageGeneration $imageGeneration */ $imageGeneration = $this->imageGenerationMapper->getImageGenerationOfImageGenId($imageGenId); } catch (Exception | DoesNotExistException | MultipleObjectsReturnedException $e) { $this->logger->warning('Image generation being deleted not in db: ' . $e->getMessage(), ['app' => Application::APP_ID]); @@ -425,14 +415,14 @@ public function cancelGeneration(string $imageGenId): void { if ($imageGeneration) { // Make sure the user is associated with the image generation - if ($imageGeneration->getUserId() !== $this->userId) { + if ($imageGeneration->getUserId() !== $userId) { $this->logger->warning('User attempted deleting another user\'s image generation!', ['app' => Application::APP_ID]); return; } // Get the generation task if it exists try { - $task = $this->textToImageManager->getUserTasksByApp($this->userId, Application::APP_ID, $imageGenId); + $task = $this->textToImageManager->getUserTasksByApp($userId, Application::APP_ID, $imageGenId); } catch (RuntimeException $e) { $this->logger->debug('Task cancellation failed or it does not exist: ' . $e->getMessage(), ['app' => Application::APP_ID]); $task = []; @@ -488,13 +478,15 @@ public function cancelGeneration(string $imageGenId): void { /** * Hide/show image files of a generation. UserId must match the assigned user of the image generation. + * * @param string $imageGenId - * @param array $fileVisSatusArray + * @param array $fileVisStatusArray + * @param string $userId * @return void + * @throws BaseException */ - public function setVisibilityOfImageFiles(string $imageGenId, array $fileVisSatusArray): void { + public function setVisibilityOfImageFiles(string $imageGenId, array $fileVisStatusArray, string $userId): void { try { - /** @var ImageGeneration $imageGeneration */ $imageGeneration = $this->imageGenerationMapper->getImageGenerationOfImageGenId($imageGenId); } catch (DoesNotExistException $e) { $this->logger->debug('Image request error : ' . $e->getMessage()); @@ -504,12 +496,12 @@ public function setVisibilityOfImageFiles(string $imageGenId, array $fileVisSatu throw new BaseException('Internal server error.', Http::STATUS_INTERNAL_SERVER_ERROR); } - if ($imageGeneration->getUserId() !== $this->userId) { + if ($imageGeneration->getUserId() !== $userId) { $this->logger->warning('User attempted deleting another user\'s image generation!'); throw new BaseException('Unauthorized.', Http::STATUS_UNAUTHORIZED); } /** @var array $fileVisStatus */ - foreach ($fileVisSatusArray as $fileVisStatus) { + foreach ($fileVisStatusArray as $fileVisStatus) { try { $this->imageFileNameMapper->setFileNameHidden(intval($fileVisStatus['id']), !((bool) $fileVisStatus['visible'])); } catch (Exception | DoesNotExistException | MultipleObjectsReturnedException $e) { @@ -521,11 +513,13 @@ public function setVisibilityOfImageFiles(string $imageGenId, array $fileVisSatu /** * Notify when image generation is ready + * * @param string $imageGenId + * @param string $userId + * @throws Exception */ - public function notifyWhenReady(string $imageGenId): void { + public function notifyWhenReady(string $imageGenId, string $userId): void { try { - /** @var ImageGeneration $imageGeneration */ $imageGeneration = $this->imageGenerationMapper->getImageGenerationOfImageGenId($imageGenId); } catch (DoesNotExistException $e) { $this->logger->debug('Image request error : ' . $e->getMessage()); @@ -544,54 +538,41 @@ public function notifyWhenReady(string $imageGenId): void { throw new BaseException('Internal server error.', Http::STATUS_INTERNAL_SERVER_ERROR); } - if ($imageGeneration->getUserId() !== $this->userId) { + if ($imageGeneration->getUserId() !== $userId) { $this->logger->warning('User attempted enabling notifications of another user\'s image generation!'); throw new BaseException('Unauthorized.', Http::STATUS_UNAUTHORIZED); } $this->imageGenerationMapper->setNotifyReady($imageGenId, true); - // Just in case the image generation is already ready, notify the user immediately so that the result is not lost: - if ($imageGeneration->getIsGenerated()) { - $this->notifyUser($imageGenId); + // Just in case check if the image generation is already ready and, if so, notify the user immediately so that the result is not lost: + try { + $tasks = $this->textToImageManager->getUserTasksByApp($userId, Application::APP_ID, $imageGenId); + } catch (RuntimeException $e) { + $this->logger->debug('Assistant meta task for the given generation id does not exist or could not be retrieved: ' . $e->getMessage(), ['app' => Application::APP_ID]); + return; } - } - /** - * Get raw image page - * @param string $imageGenId - * @return array - */ - public function getRawImagePage(string $imageGenId): array { - $generationInfo = $this->getGenerationInfo($imageGenId, true); - - /** @var array $imageFiles */ - $imageFiles = $generationInfo['files']; - - // Generate a HTML link to each image - /** @var string[] $links */ - $links = []; - /** @var array $imageFile */ - foreach ($imageFiles as $imageFile) { - $links[] = $this->urlGenerator->linkToRouteAbsolute( - Application::APP_ID . '.Text2Image.getImage', - [ - 'imageGenId' => $imageGenId, - 'fileNameId' => intval($imageFile['id']), - ] - ); + if (count($tasks) !== 1) { + $this->logger->debug('Expecting exactly one image generation task per image generation id, but found ' . count($tasks) . ' tasks.', ['app' => Application::APP_ID]); + return; } - // Create a simple http page in the response: - $body = ''; - foreach ($links as $link) { - $body .= 'image'; - $body .= '
'; + $task = array_pop($tasks); + + if ($task->getStatus() === Task::STATUS_SUCCESSFUL || $task->getStatus() === Task::STATUS_FAILED) { + // Get the assistant task + try { + $assistantTask = $this->metaTaskMapper->getMetaTaskByOcpTaskIdAndCategory($task->getId(), Application::TASK_CATEGORY_TEXT_TO_IMAGE); + } catch (Exception | DoesNotExistException | MultipleObjectsReturnedException $e) { + $this->logger->debug('Assistant meta task for the given generation id does not exist or could not be retrieved: ' . $e->getMessage(), ['app' => Application::APP_ID]); + return; + } + $assistantTask->setStatus($task->getStatus()); + // No need to update the output since it's already set + $assistantTask = $this->metaTaskMapper->update($assistantTask); + + $this->assistantService->sendNotification($assistantTask); } - $body .= ''; - return ['body' => $body, - 'headers' => [ - 'Content-Type' => ['text/html'], - ],]; } } diff --git a/psalm.xml b/psalm.xml index 42fd3b2b..5613fa39 100644 --- a/psalm.xml +++ b/psalm.xml @@ -30,11 +30,13 @@ + + diff --git a/src/assistant.js b/src/assistant.js index d9e6ec81..196a5c2d 100644 --- a/src/assistant.js +++ b/src/assistant.js @@ -1,9 +1,14 @@ import { STATUS, TASK_TYPES } from './constants.js' import { linkTo } from '@nextcloud/router' import { getRequestToken } from '@nextcloud/auth' + __webpack_nonce__ = btoa(getRequestToken()) // eslint-disable-line __webpack_public_path__ = linkTo('assistant', 'js/') // eslint-disable-line +export async function openAssistantForm(params) { + return openAssistantTextProcessingForm(params) +} + /** * Creates an assistant modal and return a promise which provides the result * @@ -36,11 +41,12 @@ __webpack_public_path__ = linkTo('assistant', 'js/') // eslint-disable-line * @param {boolean} params.isInsideViewer Should be true if this function is called while the Viewer is displayed * @param {boolean} params.closeOnResult If true, the modal will be closed when getting a sync result * @param {Array} params.actionButtons List of extra buttons to show in the assistant result form (only if closeOnResult is false) + * @param {boolean} params.useMetaTasks If true, the promise will resolve with the meta task object instead of the ocp task * @return {Promise} */ export async function openAssistantTextProcessingForm({ appId, identifier = '', taskType = null, input = '', - isInsideViewer = undefined, closeOnResult = false, actionButtons = undefined, + isInsideViewer = undefined, closeOnResult = false, actionButtons = undefined, useMetaTasks = false, }) { const { default: Vue } = await import(/* webpackChunkName: "vue-lazy" */'vue') const { default: AssistantTextProcessingModal } = await import(/* webpackChunkName: "assistant-modal-lazy" */'./components/AssistantTextProcessingModal.vue') @@ -59,8 +65,7 @@ export async function openAssistantTextProcessingForm({ const view = new View({ propsData: { isInsideViewer, - input, - taskType: TASK_TYPES.text_generation, + inputs: { prompt: input }, textProcessingTaskTypeId, showScheduleConfirmation: false, showSyncTaskRunning: false, @@ -74,13 +79,13 @@ export async function openAssistantTextProcessingForm({ reject(new Error('User cancellation')) }) view.$on('submit', (data) => { - scheduleTask(appId, identifier, data.taskTypeId, data.input) - .then((response) => { - view.input = data.input + scheduleTask(appId, identifier, data.textProcessingTaskTypeId, data.inputs) + .then(async (response) => { + view.inputs = data.inputs view.showScheduleConfirmation = true - const task = response.data?.ocs?.data?.task + const task = response.data?.task lastTask = task - resolve(task) + useMetaTasks ? resolve(task) : resolve(await resolveMetaTaskToOcpTask(task)) }) .catch(error => { view.$destroy() @@ -91,14 +96,14 @@ export async function openAssistantTextProcessingForm({ view.$on('sync-submit', (data) => { view.loading = true view.showSyncTaskRunning = true - view.input = data.input - view.textProcessingTaskTypeId = data.taskTypeId - runOrScheduleTask(appId, identifier, data.taskTypeId, data.input) - .then((response) => { + view.inputs = data.inputs + view.textProcessingTaskTypeId = data.textProcessingTaskTypeId + runOrScheduleTask(appId, identifier, data.textProcessingTaskTypeId, data.inputs) + .then(async (response) => { const task = response.data?.task lastTask = task - resolve(task) - view.input = task.input + useMetaTasks ? resolve(task) : resolve(await resolveMetaTaskToOcpTask(task)) + view.inputs = task.inputs if (task.status === STATUS.successfull) { if (closeOnResult) { view.$destroy() @@ -125,13 +130,13 @@ export async function openAssistantTextProcessingForm({ }) view.$on('cancel-sync-n-schedule', () => { cancelCurrentSyncTask() - scheduleTask(appId, identifier, view.textProcessingTaskTypeId, view.input) - .then((response) => { + scheduleTask(appId, identifier, view.textProcessingTaskTypeId, view.inputs) + .then(async (response) => { view.showSyncTaskRunning = false view.showScheduleConfirmation = true - const task = response.data?.ocs?.data?.task + const task = response.data?.task lastTask = task - resolve(task) + useMetaTasks ? resolve(task) : resolve(await resolveMetaTaskToOcpTask(task)) }) .catch(error => { view.$destroy() @@ -149,18 +154,37 @@ export async function openAssistantTextProcessingForm({ }) } +async function resolveMetaTaskToOcpTask(metaTask) { + const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') + const { generateOcsUrl } = await import(/* webpackChunkName: "router-gen-lazy" */'@nextcloud/router') + if (metaTask.category !== TASK_TYPES.text_generation) { + // For now we only resolve text generation tasks + return null + } + + const url = generateOcsUrl('textprocessing/task/{taskId}', { taskId: metaTask.ocpTaskId }) + try { + const response = await axios.get(url) + console.debug('resolved meta task', response.data?.ocs?.data?.task) + return response.data?.ocs?.data?.task + } catch (error) { + console.error(error) + return null + } +} + export async function cancelCurrentSyncTask() { window.assistantAbortController?.abort() } -export async function runTask(appId, identifier, taskType, input) { +export async function runTask(appId, identifier, taskType, inputs) { window.assistantAbortController = new AbortController() const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') const { generateUrl } = await import(/* webpackChunkName: "router-gen-lazy" */'@nextcloud/router') saveLastSelectedTaskType(taskType) - const url = generateUrl('/apps/assistant/run') + const url = generateUrl('/apps/assistant/task/run') const params = { - input, + inputs, type: taskType, appId, identifier, @@ -168,14 +192,14 @@ export async function runTask(appId, identifier, taskType, input) { return axios.post(url, params, { signal: window.assistantAbortController.signal }) } -export async function runOrScheduleTask(appId, identifier, taskType, input) { +export async function runOrScheduleTask(appId, identifier, taskType, inputs) { window.assistantAbortController = new AbortController() const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') const { generateUrl } = await import(/* webpackChunkName: "router-gen-lazy" */'@nextcloud/router') saveLastSelectedTaskType(taskType) - const url = generateUrl('/apps/assistant/run-or-schedule') + const url = generateUrl('/apps/assistant/task/run-or-schedule') const params = { - input, + inputs, type: taskType, appId, identifier, @@ -189,16 +213,16 @@ export async function runOrScheduleTask(appId, identifier, taskType, input) { * @param {string} appId the scheduling app id * @param {string} identifier the task identifier * @param {string} taskType the task type class - * @param {string} input the task input text + * @param {Array} inputs the task input texts as an array * @return {Promise<*>} */ -export async function scheduleTask(appId, identifier, taskType, input) { +export async function scheduleTask(appId, identifier, taskType, inputs) { const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') - const { generateOcsUrl } = await import(/* webpackChunkName: "router-genocs-lazy" */'@nextcloud/router') + const { generateUrl } = await import(/* webpackChunkName: "router-gen-lazy" */'@nextcloud/router') saveLastSelectedTaskType(taskType) - const url = generateOcsUrl('textprocessing/schedule', 2) + const url = generateUrl('/apps/assistant/task/schedule') const params = { - input, + inputs, type: taskType, appId, identifier, @@ -246,70 +270,36 @@ export function handleNotification(event) { // We use the object type to know if (event.notification.objectType === 'task') { event.cancelAction = true - showTextProcessingTaskResult(event.notification.objectId) - } else if (event.notification.objectType === 'speech-to-text-result') { - event.cancelAction = true - showSpeechToTextResult(event.notification) + showAssistantTaskResult(event.notification.objectId) } } /** - * Show the result of a task + * Show the result of a task based on the meta task id * - * @param {number} taskId the task id to show the result of + * @param {number} taskId the assistant meta task id to show the result of * @return {Promise} */ -async function showTextProcessingTaskResult(taskId) { +async function showAssistantTaskResult(taskId) { const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') - const { generateOcsUrl } = await import(/* webpackChunkName: "router-lazy" */'@nextcloud/router') + const { generateUrl } = await import(/* webpackChunkName: "router-lazy" */'@nextcloud/router') const { showError } = await import(/* webpackChunkName: "dialogs-lazy" */'@nextcloud/dialogs') - const url = generateOcsUrl('textprocessing/task/{taskId}', { taskId }) + const url = generateUrl('apps/assistant/task/{taskId}', { taskId }) axios.get(url).then(response => { - console.debug('showing results for task', response.data.ocs.data.task) - openAssistantTaskResult(response.data.ocs.data.task) + console.debug('showing results for task', response.data.task) + openAssistantTaskResult(response.data.task, true) }).catch(error => { console.error(error) showError(t('assistant', 'This task does not exist or has been cleaned up')) }) } -/** - * Show the result of a speech to text transcription - * @param {object} notification the notification object - * @return {Promise} - */ -async function showSpeechToTextResult(notification) { - const { default: Vue } = await import(/* webpackChunkName: "vue-lazy" */'vue') - const { showError } = await import(/* webpackChunkName: "dialogs-lazy" */'@nextcloud/dialogs') - Vue.mixin({ methods: { t, n } }) - - const { generateUrl } = await import(/* webpackChunkName: "router-lazy" */'@nextcloud/router') - const { default: axios } = await import(/* webpackChunkName: "axios-lazy" */'@nextcloud/axios') - - const params = { - params: { - id: notification.objectId, - }, - } - - const url = generateUrl('apps/assistant/stt/transcript') - - axios.get(url, params).then(response => { - console.debug('showing results for stt', response.data) - openAssistantPlainTextResult(response.data, TASK_TYPES.speech_to_text) - }).catch(error => { - console.error(error) - showError(t('assistant', 'This transcript does not exist or has been cleaned up')) - }) -} - /** * Open an assistant modal to show a plain text result - * @param {string} result the plain text result to show - * @param {number} taskType the task type + * @param {object} metaTask assistant meta task object * @return {Promise} */ -export async function openAssistantPlainTextResult(result, taskType) { +export async function openAssistantPlainTextResult(metaTask) { const { default: Vue } = await import(/* webpackChunkName: "vue-lazy" */'vue') const { default: AssistantPlainTextModal } = await import(/* webpackChunkName: "assistant-modal-lazy" */'./components/AssistantPlainTextModal.vue') Vue.mixin({ methods: { t, n } }) @@ -322,8 +312,8 @@ export async function openAssistantPlainTextResult(result, taskType) { const View = Vue.extend(AssistantPlainTextModal) const view = new View({ propsData: { - output: result, - taskType, + output: metaTask.output ?? '', + taskCategory: metaTask.category, }, }).$mount(modalElement) @@ -332,17 +322,45 @@ export async function openAssistantPlainTextResult(result, taskType) { }) } +/** + * Open an assistant modal to show an image result + * @param {object} metaTask assistant meta task object + * @return {Promise} + */ +export async function openAssistantImageResult(metaTask) { + // For now just open the image generation result on a new page: + const { generateUrl } = await import(/* webpackChunkName: "router-lazy" */'@nextcloud/router') + const url = generateUrl('apps/assistant/i/{genId}', { genId: metaTask.output }) + window.open(url, '_blank') +} + /** * Open an assistant modal to show the result of a task * * @param {object} task the task we want to see the result of + * @param {boolean} useMetaTasks If false (default), treats the input task as an ocp task, otherwise as an assistant meta task * @return {Promise} */ -export async function openAssistantTaskResult(task) { - const { showError } = await import(/* webpackChunkName: "dialogs-lazy" */'@nextcloud/dialogs') +export async function openAssistantTaskResult(task, useMetaTasks = false) { + // Divert to the right modal/page if we have a meta task with a category other than text generation: + if (useMetaTasks) { + switch (task.category) { + case TASK_TYPES.speech_to_text: + openAssistantPlainTextResult(task) + return + case TASK_TYPES.image_generation: + openAssistantImageResult(task) + return + case TASK_TYPES.text_generation: + default: + break + } + } + const { default: Vue } = await import(/* webpackChunkName: "vue-lazy" */'vue') - const { default: AssistantTextProcessingModal } = await import(/* webpackChunkName: "assistant-modal-lazy" */'./components/AssistantTextProcessingModal.vue') Vue.mixin({ methods: { t, n } }) + const { showError } = await import(/* webpackChunkName: "dialogs-lazy" */'@nextcloud/dialogs') + const { default: AssistantTextProcessingModal } = await import(/* webpackChunkName: "assistant-modal-lazy" */'./components/AssistantTextProcessingModal.vue') const modalId = 'assistantTextProcessingModal' const modalElement = document.createElement('div') @@ -353,9 +371,9 @@ export async function openAssistantTaskResult(task) { const view = new View({ propsData: { // isInsideViewer, - input: task.input, + inputs: useMetaTasks ? task.inputs : [task.input], output: task.output ?? '', - textProcessingTaskTypeId: task.type, + textProcessingTaskTypeId: useMetaTasks ? task.taskType : task.type, showScheduleConfirmation: false, }, }).$mount(modalElement) @@ -364,10 +382,10 @@ export async function openAssistantTaskResult(task) { view.$destroy() }) view.$on('submit', (data) => { - scheduleTask(task.appId, task.identifier, data.taskTypeId, data.input) + scheduleTask(task.appId, task.identifier ?? '', data.textProcessingTaskTypeId, data.inputs) .then((response) => { view.showScheduleConfirmation = true - console.debug('scheduled task', response.data?.ocs?.data?.task) + console.debug('scheduled task', response.data?.task) }) .catch(error => { view.$destroy() @@ -378,16 +396,16 @@ export async function openAssistantTaskResult(task) { view.$on('sync-submit', (data) => { view.loading = true view.showSyncTaskRunning = true - view.input = data.input - view.textProcessingTaskTypeId = data.taskTypeId - runTask(task.appId, task.identifier, data.taskTypeId, data.input) + view.inputs = data.inputs + view.textProcessingTaskTypeId = data.textProcessingTaskTypeId + runTask(task.appId, task.identifier ?? '', data.textProcessingTaskTypeId, data.inputs) .then((response) => { // resolve(response.data?.task) const task = response.data?.task if (task.status === STATUS.successfull) { view.output = task?.output } else if (task.status === STATUS.scheduled) { - view.input = task?.input + view.inputs = task?.inputs view.showScheduleConfirmation = true } view.loading = false @@ -407,7 +425,7 @@ export async function openAssistantTaskResult(task) { }) view.$on('cancel-sync-n-schedule', () => { cancelCurrentSyncTask() - scheduleTask(task.appId, task.identifier, view.textProcessingTaskTypeId, view.input) + scheduleTask(task.appId, task.identifier ?? '', view.textProcessingTaskTypeId, view.inputs) .then((response) => { view.showSyncTaskRunning = false view.showScheduleConfirmation = true @@ -437,7 +455,7 @@ export async function addAssistantMenuEntry() { }).$mount(menuEntry) view.$on('click', () => { - openAssistantTextProcessingForm({ appId: 'assistant' }) + openAssistantTextProcessingForm({ appId: 'assistant', useMetaTasks: true }) .then(r => { console.debug('scheduled task', r) }) diff --git a/src/components/AssistantFormInputs.vue b/src/components/AssistantFormInputs.vue new file mode 100644 index 00000000..d777c163 --- /dev/null +++ b/src/components/AssistantFormInputs.vue @@ -0,0 +1,214 @@ + + + + + diff --git a/src/components/AssistantPlainTextModal.vue b/src/components/AssistantPlainTextModal.vue index 273fa2e5..c9847585 100644 --- a/src/components/AssistantPlainTextModal.vue +++ b/src/components/AssistantPlainTextModal.vue @@ -6,104 +6,45 @@ @close="onCancel">
-
- - - {{ t('assistant', 'Nextcloud Assistant') }} - -

- {{ taskTypeName }} -

- - - -
- - - - {{ t('assistant', 'This output was generated by AI. Make sure to double-check and adjust.') }} - -
- - {{ t('assistant', 'Copy output') }} - - - - {{ t('assistant', 'Reset') }} - -
-
-
+ + + +
@@ -194,41 +89,6 @@ export default { //width: 100%; padding: 16px; overflow-y: auto; -} - -.assistant-modal--content { - width: 100%; - display: flex; - flex-direction: column; - align-items: center; - justify-content: center; - overflow-y: auto; - - > h2 { - display: flex; - margin: 12px 0 20px 0; - .icon { - margin-right: 8px; - } - } - - .editable-output { - width: 100%; - display: flex; - flex-direction: column; - } - - .assistant-bubble { - align-self: start; - display: flex; - gap: 8px; - background-color: var(--color-primary-element-light); - border-radius: var(--border-radius-rounded); - padding: 2px 8px; - .icon { - color: var(--color-primary); - } - } .close-button { position: absolute; @@ -239,19 +99,10 @@ export default { outline: none; } } +} - .button-wrapper { - display: flex; - flex-direction: row; - align-items: center; - justify-content: center; - margin-top: 12px; - margin-bottom: 12px; +.assistant-modal--content { + width: 100%; - >* { - margin-right: 12px; - margin-left: 12px; - } - } } diff --git a/src/components/AssistantPlainTextResult.vue b/src/components/AssistantPlainTextResult.vue new file mode 100644 index 00000000..b15c04fb --- /dev/null +++ b/src/components/AssistantPlainTextResult.vue @@ -0,0 +1,197 @@ + + + + + diff --git a/src/components/AssistantTextProcessingForm.vue b/src/components/AssistantTextProcessingForm.vue index b072b395..e36bf028 100644 --- a/src/components/AssistantTextProcessingForm.vue +++ b/src/components/AssistantTextProcessingForm.vue @@ -17,18 +17,10 @@ class="task-description"> {{ selectedTaskType.description }} - - +