feat: use identifier for the feed, change llm, update swagger

Author: luigibarbato

api/api.gen.go

@@ -29,11 +29,7 @@ paths:
                 $ref: '#/components/schemas/Error'
       x-stoplight:
         id: e6uenhk2m21k5
-      parameters:
-        - schema:
-            type: boolean
-          in: query
-          name: bySimilarity
+      parameters: []
     parameters:
       - schema:
           type: string
@@ -86,11 +82,50 @@ paths:
             schema:
               type: object
               properties: {}
+  '/v1/search/feed/similarities/{feedID}':
+    get:
+      summary: Your GET endpoint
+      tags: []
+      responses:
+        '200':
+          description: OK
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/FeedDetails'
+        '400':
+          description: Bad Request
+        '500':
+          description: Internal Server Error
+      operationId: get-v1-search-feed-similarities
+      x-stoplight:
+        id: cu8bexmv3qwv6
+      parameters: []
+    parameters:
+      - schema:
+          type: string
+        name: feedID
+        in: path
+        required: true
 components:
   schemas:
     FeedItem:
       type: object
+      x-stoplight:
+        id: kwewfmf4xgq3e
+      required:
+        - id
+        - title
+        - summary
+        - link
+        - source
+        - language
+        - date
       properties:
+        id:
+          type: string
+          x-stoplight:
+            id: i41a7r351wuiy
         title:
           type: string
         summary:
@@ -107,15 +142,10 @@ components:
         date:
           type: string
           format: date-time
-      required:
-        - title
-        - summary
-        - link
-        - source
-        - language
-        - date
-      x-stoplight:
-        id: kwewfmf4xgq3e
+        '':
+          type: string
+          x-stoplight:
+            id: 6uzhltl5f1ayg
     FeedImage:
       type: object
       properties:
@@ -209,3 +239,17 @@ components:
       required:
         - commit
         - version
+    FeedDetails:
+      title: FeedDetails
+      x-stoplight:
+        id: jdfavk4q6qd6o
+      type: object
+      properties:
+        source:
+          $ref: '#/components/schemas/FeedItem'
+        similarities:
+          type: array
+          x-stoplight:
+            id: 0s2id9mhqzyv9
+          items:
+            $ref: '#/components/schemas/FeedItem'

api/swagger.yaml

@@ -6,9 +6,10 @@ import (
 
 type FeedRepository interface {
 	// Search returns the results of a search query.
-	Find(query string) ([]Feed, error)
+	FindByKeyword(query string) ([]Feed, error)
 	// Search returns the results of a search query by similarity.
-	FindBySimilarity(query string) ([]Feed, error)
+	FindBySimilarity(doc Feed) ([]Feed, error)
+	FindByID(id string) (Feed, error)
 	// Index indexes a document.
 	Save(doc Feed) error
 	// Update a document in index.
@@ -20,6 +21,7 @@ type FeedRepository interface {
 }
 
 type Feed struct {
+	ID       string    `json:"id"`
 	Title    string    `json:"title"`
 	Link     string    `json:"link"`
 	Language string    `json:"language"`

internal/app/feed.go

@@ -2,6 +2,9 @@ package typesense
 
 import (
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
 	"time"
 
 	"github.com/typesense/typesense-go/typesense"
@@ -21,7 +24,7 @@ func NewFeedRepository(client *typesense.Client) *FeedRepository {
 	}
 }
 
-func (f *FeedRepository) Find(query string) ([]app.Feed, error) {
+func (f *FeedRepository) FindByKeyword(query string) ([]app.Feed, error) {
 	searchParameters := &api.SearchCollectionParams{
 		Q:       query,
 		QueryBy: "title, summary",
@@ -41,6 +44,7 @@ func (f *FeedRepository) Find(query string) ([]app.Feed, error) {
 		}
 
 		f := app.Feed{
+			ID:       doc["id"].(string),
 			Title:    doc["title"].(string),
 			Link:     doc["link"].(string),
 			Source:   doc["source"].(string),
@@ -55,11 +59,38 @@ func (f *FeedRepository) Find(query string) ([]app.Feed, error) {
 	return feeds, nil
 }
 
+func (f *FeedRepository) FindByID(id string) (app.Feed, error) {
+	d, err := f.client.Collection("feeds").Document(id).Retrieve(f.ctx)
+	if err != nil {
+		return app.Feed{}, err
+	}
+
+	date, err := time.Parse(time.RFC3339, d["date"].(string))
+	if err != nil {
+		return app.Feed{}, err
+	}
+
+	fStruct := app.Feed{
+		ID:       d["id"].(string),
+		Title:    d["title"].(string),
+		Link:     d["link"].(string),
+		Source:   d["source"].(string),
+		Language: d["language"].(string),
+		Summary:  d["summary"].(string),
+		Date:     date,
+	}
 
-func (f *FeedRepository) FindBySimilarity(query string) ([]app.Feed, error){
+	return fStruct, nil
+}
+
+func (f *FeedRepository) FindBySimilarity(feed app.Feed) ([]app.Feed, error) {
+	vQ := fmt.Sprintf("title_summary_embedding:([], id: %s, distance_threshold:0.183)", feed.ID)
+	eF := "title_summary_embedding"
 	searchParameters := &api.SearchCollectionParams{
-		Q:       query,
-		QueryBy: "title_summary_embedding",
+		Q:             feed.Title,
+		QueryBy:       "title,title_summary_embedding",
+		VectorQuery:   &vQ,
+		ExcludeFields: &eF,
 	}
 	searchResult, err := f.client.Collection("feeds").Documents().Search(f.ctx, searchParameters)
 	if err != nil {
@@ -90,10 +121,9 @@ func (f *FeedRepository) FindBySimilarity(query string) ([]app.Feed, error){
 	return feeds, nil
 }
 
-
 func (f *FeedRepository) Save(doc app.Feed) error {
 	docMap := map[string]interface{}{
-		"id":       doc.Link,
+		"id":       generateUniqueID(doc.Link),
 		"title":    doc.Title,
 		"link":     doc.Link,
 		"source":   doc.Source,
@@ -117,7 +147,7 @@ func (f *FeedRepository) Update(docs ...app.Feed) error {
 	for i, doc := range docs {
 		// Convert app.Feed to map[string]interface{} for updating
 		docMap := map[string]interface{}{
-			"id":       doc.Link,
+			"id":       generateUniqueID(doc.Link),
 			"title":    doc.Title,
 			"link":     doc.Link,
 			"source":   doc.Source,
@@ -160,3 +190,13 @@ func (f *FeedRepository) Delete(doc app.Feed) error {
 
 	return nil
 }
+
+func generateUniqueID(link string) string {
+	hash := sha256.New()
+	hash.Write([]byte(link))
+	hashBytes := hash.Sum(nil)
+
+	uniqueID := hex.EncodeToString(hashBytes)
+
+	return uniqueID
+}

internal/repository/typesense/feed.go

@@ -51,7 +51,7 @@ func GetFeedSchema(client *typesense.Client) *api.CollectionSchema {
 						ModelName    string  "json:\"model_name\""
 						ProjectId    *string "json:\"project_id,omitempty\""
 					}{
-						ModelName: "ts/all-MiniLM-L12-v2",
+						ModelName: "ts/multilingual-e5-large",
 					},
 				},
 			},