From 53db6677cea6066ab8fd65099abbfef257db6e98 Mon Sep 17 00:00:00 2001 From: joel-joseph-george Date: Mon, 10 Mar 2025 14:29:43 +0530 Subject: [PATCH 1/5] Adding docker --- UI/.dockerignore | 2 ++ UI/Dockerfile | 15 ++++++++++++++ docker/docker-compose.yml | 42 +++++++++++++++++++++++++++++---------- 3 files changed, 48 insertions(+), 11 deletions(-) create mode 100644 UI/.dockerignore create mode 100644 UI/Dockerfile diff --git a/UI/.dockerignore b/UI/.dockerignore new file mode 100644 index 0000000..329d75c --- /dev/null +++ b/UI/.dockerignore @@ -0,0 +1,2 @@ +node_modules +.env.local \ No newline at end of file diff --git a/UI/Dockerfile b/UI/Dockerfile new file mode 100644 index 0000000..2dbdc74 --- /dev/null +++ b/UI/Dockerfile @@ -0,0 +1,15 @@ +FROM node:18-alpine + +WORKDIR /app + +COPY package*.json ./ + +RUN npm install + +COPY . . + +RUN npm run build + +EXPOSE 3001 + +CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 0c1fd49..73f4f8a 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,15 +40,35 @@ services: - gitea volumes: - ./postgres:/var/lib/postgresql/data + ui: + # image: joeljosephgeorge/gitea-ui:v1.0.3 + build: + context: ../UI + dockerfile: Dockerfile + healthcheck: + timeout: 45s + interval: 10s + retries: 10 + container_name: gitea-ui - #Uncomment the following lines to enable the gitea-ui - # ui: - # image: joeljosephgeorge/gitea-ui:v1.0.0 - # container_name: gitea-ui - # restart: always - # networks: - # - gitea - # ports: - # - "3001:3001" - # depends_on: - # - server + # env_file: + # - .env.local + environment: + - VITE_GITEA_BASE_URL=http://localhost:3000 + - VITE_GITEA_UI_URL=http://localhost:3001 + - VITE_GITEA_ORG_NAME=BCS + - VITE_FASTAPI_BASE_URL=http://localhost:8000 + #The below values are dummy values, replace them with your own values + #the values where not taking when running the api and ui in docker, i could run it by adding + #them in the .env.local file and adding the file in the env_file section + - VITE_GITEA_CLIENT_ID = {your-client-id} + - VITE_GITEA_CLIENT_SECRET = {your-client-secret} + - VITE_GITEA_REDIRECT_URI=http://localhost:3001/oauth/callback + + restart: always + networks: + - gitea + ports: + - "3001:3001" + depends_on: + - server From c6291dfd1e7ac8d95dd537cdeaccc721372b746b Mon Sep 17 00:00:00 2001 From: joel-joseph-george Date: Mon, 10 Mar 2025 14:30:27 +0530 Subject: [PATCH 2/5] Adding UI changes ( SFM support) --- UI/src/pages/ViewFile.tsx | 11 +++++++--- .../data-preprocessing/ParallelCorpora.tsx | 20 ++++++++++--------- .../pages/data-preprocessing/ParseBooks.tsx | 5 +++-- 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/UI/src/pages/ViewFile.tsx b/UI/src/pages/ViewFile.tsx index eb26be0..7d8c09a 100644 --- a/UI/src/pages/ViewFile.tsx +++ b/UI/src/pages/ViewFile.tsx @@ -53,8 +53,10 @@ export const ViewFile: React.FC = () => { : (fileContent.size / 1024).toFixed(2) + " KB"; // Determine whether "Copy" and "Lines" are enabled - const isCopyEnabled = ["txt", "md", "usfm", "csv"].includes(fileExtension); - const isLineEnabled = ["txt", "pdf", "md", "usfm", "csv"].includes( + const isCopyEnabled = ["txt", "md", "usfm", "sfm", "csv"].includes( + fileExtension + ); + const isLineEnabled = ["txt", "pdf", "md", "usfm", "sfm", "csv"].includes( fileExtension ); const isDownloadEnabled = ["pdf"].includes(fileExtension); @@ -187,10 +189,13 @@ export const ViewFile: React.FC = () => { ); - } else if (["txt", "usfm", "json"].includes(fileExtension)) { + } else if (["txt", "usfm", "sfm", "json"].includes(fileExtension)) { return ( { const bibles = response?.data?.bibles || []; // Calculate the total successful books across all bibles - const successfulBooksCount = bibles.reduce((total, bible) => { + const successfulBooksCount = bibles.reduce((total: any, bible: any) => { // Check if bible has books array if (bible.books && Array.isArray(bible.books)) { // Count only the books with status "success" const successfulBooks = bible.books.filter( - (book) => book.status === "success" + (book: any) => book.status === "success" ); return total + successfulBooks.length; } @@ -133,10 +133,10 @@ const ParallelCorpora: React.FC = () => { // Same logic for repo2 const bibles = response?.data?.bibles || []; - const successfulBooksCount = bibles.reduce((total, bible) => { + const successfulBooksCount = bibles.reduce((total: any, bible: any) => { if (bible.books && Array.isArray(bible.books)) { const successfulBooks = bible.books.filter( - (book) => book.status === "success" + (book: any) => book.status === "success" ); return total + successfulBooks.length; } @@ -175,8 +175,8 @@ const ParallelCorpora: React.FC = () => { try { // Determine which API endpoint to use based on withBCV const apiEndpoint = withBCV - ? `${import.meta.env.VITE_FASTAPI_BASE_URL}/parallel_corpora/withbcv/csv/` - : `${import.meta.env.VITE_FASTAPI_BASE_URL}/parallel_corpora/withoutbcv/csv/`; + ? `${import.meta.env.VITE_FASTAPI_BASE_URL}/parallel_corpora/withbcv/` + : `${import.meta.env.VITE_FASTAPI_BASE_URL}/parallel_corpora/withoutbcv/`; // Make the API call const response = await axios.get(apiEndpoint, { @@ -193,12 +193,14 @@ const ParallelCorpora: React.FC = () => { // Create a link element and trigger download const link = document.createElement("a"); link.href = window.URL.createObjectURL(blob); - const fileName = withBCV ? `parallel_corpora_BCV_${selectedProject1}_${selectedProject2}.csv`: `parallel_corpora_${selectedProject1}_${selectedProject2}.csv`; + const fileName = withBCV + ? `parallel_corpora_BCV_${selectedProject1}_${selectedProject2}.csv` + : `parallel_corpora_${selectedProject1}_${selectedProject2}.csv`; link.download = fileName; document.body.appendChild(link); link.click(); document.body.removeChild(link); - } catch (error) { + } catch (error: any) { console.error("Error downloading CSV:", error); toast({ variant: "destructive", @@ -252,7 +254,7 @@ const ParallelCorpora: React.FC = () => { handleSelectValueChange("repo2", value) } options={projects.filter( - (project) => project.name !== selectedProject1 + (project: any) => project.name !== selectedProject1 )} placeholder="Select target project" /> diff --git a/UI/src/pages/data-preprocessing/ParseBooks.tsx b/UI/src/pages/data-preprocessing/ParseBooks.tsx index dea8874..e551a30 100644 --- a/UI/src/pages/data-preprocessing/ParseBooks.tsx +++ b/UI/src/pages/data-preprocessing/ParseBooks.tsx @@ -89,8 +89,9 @@ const ParseBooks = ({ owner, repo }: ParseBooksProps) => { const filesResponse = await API.get( `/api/v1/repos/${owner}/${repo}/contents/` ); - const usfmFiles = filesResponse.data.filter((file: { name: string }) => - file.name.endsWith(".usfm") + const usfmFiles = filesResponse.data.filter( + (file: { name: string }) => + file.name.endsWith(".usfm") || file.name.endsWith(".SFM") ); setUsfmCount(usfmFiles.length); if (usfmFiles.length === 0) return; From 8acba39dc54aa250ba78c170d786fd1827f3c02d Mon Sep 17 00:00:00 2001 From: joel-joseph-george Date: Tue, 11 Mar 2025 10:20:53 +0530 Subject: [PATCH 3/5] Adding documentation --- docker/Readme.md | 57 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 docker/Readme.md diff --git a/docker/Readme.md b/docker/Readme.md new file mode 100644 index 0000000..cf7e0b5 --- /dev/null +++ b/docker/Readme.md @@ -0,0 +1,57 @@ +# Gitea Docker Setup Guide + +## Introduction + +This guide will help you set up Gitea using Docker. Gitea is a lightweight Git service that can be self-hosted. This guide assumes you have Docker and Docker Compose installed on your system. + +## Prerequisites + +- Docker +- Docker Compose + +## Steps to Install Gitea + +1. **Clone the Repository** + Clone the repository containing the Docker Compose file and other necessary configurations. + + ```sh + git clone https://github.com/Bridgeconn/ai_mft_data_preprocessing.git + cd ai_mft_data_preprocessing + ``` + +2. **Run Docker Compose** + Start the Gitea service using Docker Compose. + + ```sh + cd docker + docker compose up -d + ``` + +3. **Initial Setup of Gitea** + When you first boot the Gitea image, you will be prompted to configure the database. Since the database configuration is already included in the Docker Compose file, you can proceed with the following steps: + + - **Database Configuration**: This step will be pre-configured based on your Docker Compose settings. + - **SMTP Configuration**: You can skip this step if you do not need email notifications. + - **Root Admin Creation**: This can be done later when you create the first user. + +4. **Complete Installation** + Click the "Install" button to complete the setup. Gitea will now be running. + +5. **Accessing the Gitea** + Open your web browser and navigate to the URL where Gitea is hosted. Refer to the UI documentation for detailed instructions on how to use the Gitea-UI and how you can setup the Oauth configuration. + +## Additional Documentation + +For more detailed instructions on using the Gitea UI, please refer to the [Gitea UI Documentation](https://github.com/Bridgeconn/ai_mft_data_preprocessing/tree/main/UI/docs/Installation). + +## Adding variables + +Once you have created a new application for oauth in the gitea, you will be having a `client-id` and `client-secret` . Make sure you add them to our docker compose file. Rest for the variables are already added. + +## Note + +Once you created the backend(Gitea), create a Organization named `BCS`. and make sure if you are creating repositories in the backend, create under the organization while if you are using the UI , then you dont have to worry on that. + +## Conclusion + +You have successfully set up Gitea using Docker. You can now start using Gitea for your Git repositories. From 91dea5aea71c8498e95cc7db42136e5916e47822 Mon Sep 17 00:00:00 2001 From: AthulyaMS Date: Tue, 11 Mar 2025 18:48:08 +0530 Subject: [PATCH 4/5] updated docker env ,added dockerfile inside docker folder,updated readme with docker steps --- UI/Dockerfile | 15 ------------- UI/README.md | 47 +++++++++++++++++++++++++++++++++++++++ docker/Dockerfile | 12 ++++++++++ docker/docker-compose.yml | 27 +++++++++------------- 4 files changed, 70 insertions(+), 31 deletions(-) delete mode 100644 UI/Dockerfile create mode 100644 docker/Dockerfile diff --git a/UI/Dockerfile b/UI/Dockerfile deleted file mode 100644 index 2dbdc74..0000000 --- a/UI/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM node:18-alpine - -WORKDIR /app - -COPY package*.json ./ - -RUN npm install - -COPY . . - -RUN npm run build - -EXPOSE 3001 - -CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0"] \ No newline at end of file diff --git a/UI/README.md b/UI/README.md index 83417fe..e5227f5 100644 --- a/UI/README.md +++ b/UI/README.md @@ -265,3 +265,50 @@ Key Highlights and Recommendations: - Implement proper authentication flows Would you like me to elaborate on any specific aspect of the project setup or discuss more detailed implementation strategies for Gitea integration? + + +## To run the project using docker +Ensure `.env` file is created in the docker folder with following variables. + ```bash + VITE_GITEA_BASE_URL=http://localhost:3000 + VITE_GITEA_UI_URL=http://localhost:3001 + VITE_GITEA_ORG_NAME=BCS + VITE_GITEA_CLIENT_ID = {your-client-id} + VITE_GITEA_CLIENT_SECRET = {your-client-secret} + VITE_GITEA_REDIRECT_URI=http://localhost:3001/oauth/callback + VITE_FASTAPI_BASE_URL=http://localhost:8000 + + ``` + +From the `cd ai_mft_data_preprocessing/docker` folder: + + ```bash + docker-compose up --build + ``` + +To run the containers in detached mode + + ```bash + docker-compose up --build -d + ``` + +To check logs from your running Docker containers: + + ```bash + docker logs + ``` + +To stop the App + + ```bash + docker-compose down + + ``` +If you get cors error while calling api ,add the following lines in the docker/gitea/gitea/conf/app.ini + + ```bash + [cors] + ENABLED = true + ALLOW_DOMAIN = * + + ``` \ No newline at end of file diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..b02791d --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,12 @@ +FROM node:18-alpine + +WORKDIR /app + +COPY ../UI/package*.json ./ +RUN npm install + +COPY ../UI . +RUN npm run build + +EXPOSE 3001 +CMD ["npm", "run", "dev", "--", "--host", "0.0.0.0"] diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 73f4f8a..30a566d 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -40,31 +40,26 @@ services: - gitea volumes: - ./postgres:/var/lib/postgresql/data + + ui: - # image: joeljosephgeorge/gitea-ui:v1.0.3 build: - context: ../UI - dockerfile: Dockerfile + context: ../ + dockerfile: ./docker/Dockerfile healthcheck: timeout: 45s interval: 10s retries: 10 container_name: gitea-ui - # env_file: - # - .env.local environment: - - VITE_GITEA_BASE_URL=http://localhost:3000 - - VITE_GITEA_UI_URL=http://localhost:3001 - - VITE_GITEA_ORG_NAME=BCS - - VITE_FASTAPI_BASE_URL=http://localhost:8000 - #The below values are dummy values, replace them with your own values - #the values where not taking when running the api and ui in docker, i could run it by adding - #them in the .env.local file and adding the file in the env_file section - - VITE_GITEA_CLIENT_ID = {your-client-id} - - VITE_GITEA_CLIENT_SECRET = {your-client-secret} - - VITE_GITEA_REDIRECT_URI=http://localhost:3001/oauth/callback - + - VITE_GITEA_BASE_URL=${VITE_GITEA_BASE_URL} + - VITE_GITEA_UI_URL=${VITE_GITEA_UI_URL} + - VITE_GITEA_ORG_NAME=${VITE_GITEA_ORG_NAME} + - VITE_GITEA_CLIENT_ID=${VITE_GITEA_CLIENT_ID} + - VITE_GITEA_CLIENT_SECRET=${VITE_GITEA_CLIENT_SECRET} + - VITE_GITEA_REDIRECT_URI=${VITE_GITEA_REDIRECT_URI} + - VITE_FASTAPI_BASE_URL=${VITE_FASTAPI_BASE_URL} restart: always networks: - gitea From daa848cf40ef8c14833bdd605515fbfdf5167a96 Mon Sep 17 00:00:00 2001 From: Ketan Kumar Baboo Date: Fri, 28 Mar 2025 17:26:51 +0530 Subject: [PATCH 5/5] implement usfm bible book preview in the dialog modal --- .../pages/data-preprocessing/ParseBooks.tsx | 259 ++++++++++++------ 1 file changed, 180 insertions(+), 79 deletions(-) diff --git a/UI/src/pages/data-preprocessing/ParseBooks.tsx b/UI/src/pages/data-preprocessing/ParseBooks.tsx index e551a30..f6b0902 100644 --- a/UI/src/pages/data-preprocessing/ParseBooks.tsx +++ b/UI/src/pages/data-preprocessing/ParseBooks.tsx @@ -3,6 +3,7 @@ import { API } from "@/services/Api"; import axios from "axios"; import { useState } from "react"; import { Dialog, DialogContent, DialogFooter } from "@/components/ui/dialog"; +import { Eye } from "lucide-react"; interface ParseBooksProps { owner: string; @@ -16,13 +17,25 @@ interface ParseResult { details?: any; } +interface BookPreview { + project_name: string; + book_name: string; + chapters: Array<{ + chapter: string; + verses: Array<{ + verse: string; + text: string; + }>; + }>; +} + const ParseBooks = ({ owner, repo }: ParseBooksProps) => { const [loading, setLoading] = useState(false); const [isDialogOpen, setIsDialogOpen] = useState(false); const [parseResults, setParseResults] = useState([]); - const [selectedResult, setSelectedResult] = useState( - null - ); + const [selectedResult, setSelectedResult] = useState(null); + const [bookPreview, setBookPreview] = useState(null); + const [isPreviewMode, setIsPreviewMode] = useState(false); const [usfmCount, setUsfmCount] = useState(0); const [parseCount, setParseCount] = useState(0); @@ -38,12 +51,26 @@ const ParseBooks = ({ owner, repo }: ParseBooksProps) => { } }; + const fetchBookPreview = async (bookName: string) => { + try { + const response = await axios.get( + `${import.meta.env.VITE_FASTAPI_BASE_URL}/book/json/?project_name=${repo}&book_name=${bookName}` + ); + console.log("response of book data", response.data); + setBookPreview(response.data); + setIsPreviewMode(true); + } catch (error) { + console.error("Error fetching book preview:", error); + } + }; + const usfmParseFunction = async (file: any, method: "post" | "put") => { try { const contentResponse = await API.get( `/api/v1/repos/${owner}/${repo}/contents/${file.path}` ); const responseData = contentResponse.data.content; + console.log("file", file, "method", method); if (responseData) { try { const uploadResponse = await axios[method]( @@ -82,7 +109,7 @@ const ParseBooks = ({ owner, repo }: ParseBooksProps) => { setLoading(true); const results: any = []; const listBooks = await fetchListBibles(); - // const filterBooksSuccess = listBooks.filter( + // const filterBooksSuccess = listBooks.filter( // (book: any) => book.status === "success" // ); try { @@ -99,9 +126,11 @@ const ParseBooks = ({ owner, repo }: ParseBooksProps) => { for (const file of usfmFiles) { setParseCount((prevCount) => prevCount + 1); const fileNameUpperCase = file.name?.split(".")[0].toUpperCase(); + console.log("list of books", listBooks) const bookExists = listBooks.some( (book: any) => book.book_name === fileNameUpperCase ); + console.log("book exists", bookExists) const method = bookExists ? "put" : "post"; const parseUSFMResponse = await usfmParseFunction(file, method); results.push(parseUSFMResponse); @@ -110,6 +139,7 @@ const ParseBooks = ({ owner, repo }: ParseBooksProps) => { setParseResults(results); setSelectedResult(results.length > 0 ? results[0] : null); setIsDialogOpen(true); + setIsPreviewMode(false); } catch (error) { console.error("Error fetching books:", error); } finally { @@ -119,6 +149,130 @@ const ParseBooks = ({ owner, repo }: ParseBooksProps) => { const handleResultClick = (result: ParseResult) => { setSelectedResult(result); + setIsPreviewMode(false); + }; + + const handlePreviewClick = (result: ParseResult) => { + const cleanBookName = result.bookName.split('.')[0]; + fetchBookPreview(cleanBookName); + }; + + const renderDialogContent = () => { + if (isPreviewMode && bookPreview) { + return ( +
+
+

{bookPreview.book_name}

+
+
+ {bookPreview.chapters.map((chapter) => ( +
+

+ Chapter {chapter.chapter} +

+ {chapter.verses.map((verse, index) => ( +
+ + {verse.verse} + + {verse.text} +
+ ))} +
+ ))} +
+
+ ); + } + + return ( +
+
+
+
Book Name
+
Status
+
Preview
+
+
+ {parseResults.map((result, index) => ( +
handleResultClick(result)} + > +
{result.bookName}
+
+ {result.status} +
+ {result.status === "Success" ? ( +
{ + e.stopPropagation(); + handlePreviewClick(result); + }} + > + +
+ ): ( +
+ )} +
+ ))} +
+
+
+ {selectedResult && ( +
+ {selectedResult.status === "Error" ? ( + <> +

+ {selectedResult.bookName?.split(".")[0]} +

+
+ {selectedResult.message?.message} +
+ {selectedResult.details[0] + ?.split("\n") + .map((line: string, index: number) => ( +
+ {line} +
+ ))} + + ) : ( + <> +

+ {selectedResult.bookName?.split(".")[0]} +

+
+ {selectedResult.message} +
+ + )} +
+ )} +
+
+ ); }; return ( @@ -129,82 +283,29 @@ const ParseBooks = ({ owner, repo }: ParseBooksProps) => { -
-
-
-
Book Name
-
Status
-
-
- {parseResults.map((result, index) => ( -
handleResultClick(result)} - > -
{result.bookName}
-
- {result.status} -
-
- ))} -
-
-
- {selectedResult && ( -
- {selectedResult.status === "Error" ? ( - <> -

- {selectedResult.bookName?.split(".")[0]} -

-
- {selectedResult.message?.message} -
- {selectedResult.details[0] - ?.split("\n") - .map((line: string, index: number) => ( -
- {line} -
- ))} - - ) : ( - <> -

- {selectedResult.bookName?.split(".")[0]} -

-
- {selectedResult.message} -
- - )} -
- )} -
-
+ {renderDialogContent()} - + {isPreviewMode ? ( + + ) : ( + + )} @@ -212,4 +313,4 @@ const ParseBooks = ({ owner, repo }: ParseBooksProps) => { ); }; -export default ParseBooks; +export default ParseBooks; \ No newline at end of file