Skip to content

Commit 722d990

Browse files
hunnyboy1217hunnyboy1217
committed
feat[Go]: implement chunk REST APIs (list, add, update, switch)
Ports four missing chunk endpoints from Python (chunk_api.py) to Go, closing issue #15668: GET /api/v1/datasets/:dataset_id/documents/:document_id/chunks (ListChunksREST) POST /api/v1/datasets/:dataset_id/documents/:document_id/chunks (AddChunk) PATCH /api/v1/datasets/:dataset_id/documents/:document_id/chunks/:chunk_id (UpdateChunkREST) PATCH /api/v1/datasets/:dataset_id/documents/:document_id/chunks (SwitchChunks) Already implemented (unchanged): GET ...chunks/:chunk_id (Get) DELETE ...chunks (RemoveChunks) Changed files: internal/service/chunk.go – resolveDatasetAccess, getEmbeddingModelForKB, embedTexts, weightedVec helpers; ListChunksREST, AddChunk, UpdateChunkREST, SwitchChunks methods internal/handler/chunk.go – four new HTTP handlers internal/router/router.go – route registration Functional parity with Python: ListChunksREST – ownership check (dataset + document), page/size/keywords/available query params, doc metadata in response AddChunk – xxhash64(content+docID) chunk ID, tokenisation, 0.1*docVec + 0.9*contentVec embedding, InsertChunks, document chunk_num/token_num increment via gorm.Expr UpdateChunkREST – ownership + chunk-belongs-to-doc check, re-embedding when content or questions change (same weighted average as Python) SwitchChunks – validates chunk_ids + available, bulk UpdateChunks on engine
1 parent 461c190 commit 722d990

3 files changed

Lines changed: 679 additions & 1 deletion

File tree

‎internal/handler/chunk.go‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -337,6 +337,197 @@ func (h *ChunkHandler) UpdateChunk(c *gin.Context) {
337337
})
338338
}
339339

340+
// ListChunksREST lists chunks for a document inside a dataset.
341+
// @Summary List Chunks
342+
// @Description List chunks for a document (dataset_id and document_id from path).
343+
// @Tags chunks
344+
// @Produce json
345+
// @Param dataset_id path string true "Dataset ID"
346+
// @Param document_id path string true "Document ID"
347+
// @Param page query int false "Page number (default 1)"
348+
// @Param page_size query int false "Items per page (default 30)"
349+
// @Param keywords query string false "Keyword filter"
350+
// @Param available query bool false "Filter by available status"
351+
// @Success 200 {object} map[string]interface{}
352+
// @Router /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks [get]
353+
func (h *ChunkHandler) ListChunksREST(c *gin.Context) {
354+
user, errorCode, errorMessage := GetUser(c)
355+
if errorCode != common.CodeSuccess {
356+
jsonError(c, errorCode, errorMessage)
357+
return
358+
}
359+
360+
datasetID := c.Param("dataset_id")
361+
documentID := c.Param("document_id")
362+
if datasetID == "" || documentID == "" {
363+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": "dataset_id and document_id are required"})
364+
return
365+
}
366+
367+
page := 1
368+
if v := c.Query("page"); v != "" {
369+
if p, err := json.Number(v).Int64(); err == nil && p > 0 {
370+
page = int(p)
371+
}
372+
}
373+
pageSize := 30
374+
if v := c.Query("page_size"); v != "" {
375+
if ps, err := json.Number(v).Int64(); err == nil && ps > 0 {
376+
if ps > 100 {
377+
ps = 100
378+
}
379+
pageSize = int(ps)
380+
}
381+
}
382+
keywords := c.Query("keywords")
383+
384+
var available *bool
385+
if v := c.Query("available"); v != "" {
386+
b := v == "true" || v == "1"
387+
available = &b
388+
}
389+
390+
resp, err := h.chunkService.ListChunksREST(datasetID, documentID, user.ID, page, pageSize, keywords, available)
391+
if err != nil {
392+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": err.Error()})
393+
return
394+
}
395+
396+
c.JSON(http.StatusOK, gin.H{"code": common.CodeSuccess, "data": resp, "message": "success"})
397+
}
398+
399+
// AddChunk adds a manually created chunk to a document.
400+
// @Summary Add Chunk
401+
// @Description Create a new chunk for a document with content, keywords, and questions.
402+
// @Tags chunks
403+
// @Accept json
404+
// @Produce json
405+
// @Param dataset_id path string true "Dataset ID"
406+
// @Param document_id path string true "Document ID"
407+
// @Param request body service.AddChunkRequest true "chunk content"
408+
// @Success 200 {object} map[string]interface{}
409+
// @Router /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks [post]
410+
func (h *ChunkHandler) AddChunk(c *gin.Context) {
411+
user, errorCode, errorMessage := GetUser(c)
412+
if errorCode != common.CodeSuccess {
413+
jsonError(c, errorCode, errorMessage)
414+
return
415+
}
416+
417+
datasetID := c.Param("dataset_id")
418+
documentID := c.Param("document_id")
419+
if datasetID == "" || documentID == "" {
420+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": "dataset_id and document_id are required"})
421+
return
422+
}
423+
424+
var req service.AddChunkRequest
425+
if err := c.ShouldBindJSON(&req); err != nil {
426+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": err.Error()})
427+
return
428+
}
429+
430+
data, err := h.chunkService.AddChunk(datasetID, documentID, user.ID, &req)
431+
if err != nil {
432+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": err.Error()})
433+
return
434+
}
435+
436+
c.JSON(http.StatusOK, gin.H{"code": common.CodeSuccess, "data": data, "message": "success"})
437+
}
438+
439+
// UpdateChunkREST updates a chunk's content, keywords, and availability.
440+
// Re-embeds the chunk when content or questions change.
441+
// @Summary Update Chunk (REST)
442+
// @Description Partially update a chunk by ID, re-embedding on content/question changes.
443+
// @Tags chunks
444+
// @Accept json
445+
// @Produce json
446+
// @Param dataset_id path string true "Dataset ID"
447+
// @Param document_id path string true "Document ID"
448+
// @Param chunk_id path string true "Chunk ID"
449+
// @Param request body service.UpdateChunkRESTRequest true "fields to update"
450+
// @Success 200 {object} map[string]interface{}
451+
// @Router /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks/{chunk_id} [patch]
452+
func (h *ChunkHandler) UpdateChunkREST(c *gin.Context) {
453+
user, errorCode, errorMessage := GetUser(c)
454+
if errorCode != common.CodeSuccess {
455+
jsonError(c, errorCode, errorMessage)
456+
return
457+
}
458+
459+
datasetID := c.Param("dataset_id")
460+
documentID := c.Param("document_id")
461+
chunkID := c.Param("chunk_id")
462+
if datasetID == "" || documentID == "" || chunkID == "" {
463+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": "dataset_id, document_id and chunk_id are required"})
464+
return
465+
}
466+
467+
var req service.UpdateChunkRESTRequest
468+
if err := c.ShouldBindJSON(&req); err != nil {
469+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": err.Error()})
470+
return
471+
}
472+
473+
if err := h.chunkService.UpdateChunkREST(datasetID, documentID, chunkID, user.ID, &req); err != nil {
474+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": err.Error()})
475+
return
476+
}
477+
478+
c.JSON(http.StatusOK, gin.H{"code": common.CodeSuccess, "data": true, "message": "success"})
479+
}
480+
481+
// SwitchChunks bulk-toggles the available status for a list of chunks.
482+
// @Summary Switch Chunks Availability
483+
// @Description Toggle available_int for a set of chunk IDs.
484+
// @Tags chunks
485+
// @Accept json
486+
// @Produce json
487+
// @Param dataset_id path string true "Dataset ID"
488+
// @Param document_id path string true "Document ID"
489+
// @Param request body object true "chunk_ids + available"
490+
// @Success 200 {object} map[string]interface{}
491+
// @Router /api/v1/datasets/{dataset_id}/documents/{document_id}/chunks [patch]
492+
func (h *ChunkHandler) SwitchChunks(c *gin.Context) {
493+
user, errorCode, errorMessage := GetUser(c)
494+
if errorCode != common.CodeSuccess {
495+
jsonError(c, errorCode, errorMessage)
496+
return
497+
}
498+
499+
datasetID := c.Param("dataset_id")
500+
documentID := c.Param("document_id")
501+
if datasetID == "" || documentID == "" {
502+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": "dataset_id and document_id are required"})
503+
return
504+
}
505+
506+
var body struct {
507+
ChunkIDs []string `json:"chunk_ids"`
508+
Available *bool `json:"available"`
509+
}
510+
if err := c.ShouldBindJSON(&body); err != nil {
511+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": err.Error()})
512+
return
513+
}
514+
if len(body.ChunkIDs) == 0 {
515+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": "`chunk_ids` is required."})
516+
return
517+
}
518+
if body.Available == nil {
519+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": "`available` is required."})
520+
return
521+
}
522+
523+
if err := h.chunkService.SwitchChunks(datasetID, documentID, user.ID, body.ChunkIDs, *body.Available); err != nil {
524+
c.JSON(http.StatusOK, gin.H{"code": common.CodeDataError, "data": false, "message": err.Error()})
525+
return
526+
}
527+
528+
c.JSON(http.StatusOK, gin.H{"code": common.CodeSuccess, "data": true, "message": "success"})
529+
}
530+
340531
// RemoveChunks handles chunk removal requests
341532
// @Summary Remove Chunks
342533
// @Description Remove chunks from a document

‎internal/router/router.go‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -247,10 +247,14 @@ func (r *Router) Setup(engine *gin.Engine) {
247247
datasets.DELETE("/:dataset_id/documents", r.documentHandler.DeleteDocuments)
248248

249249
// Dataset document chunk
250+
datasets.GET("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.ListChunksREST)
251+
datasets.POST("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.AddChunk)
250252
datasets.GET("/:dataset_id/documents/:document_id/chunks/:chunk_id", r.chunkHandler.Get)
253+
datasets.PATCH("/:dataset_id/documents/:document_id/chunks/:chunk_id", r.chunkHandler.UpdateChunkREST)
254+
datasets.PATCH("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.SwitchChunks)
255+
datasets.DELETE("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.RemoveChunks)
251256
datasets.POST("/:dataset_id/documents/parse", r.documentHandler.ParseDocuments)
252257
datasets.POST("/:dataset_id/documents/stop", r.documentHandler.StopParseDocuments)
253-
datasets.DELETE("/:dataset_id/documents/:document_id/chunks", r.chunkHandler.RemoveChunks)
254258
}
255259

256260
// Search routes

0 commit comments

Comments
 (0)