Implement complete collection persistence with database-backed survival across server restarts

• Add full multi-table schema for collections with normalized design (collections, collection_templates, collection_items, collection_item_versions)
• Implement collection detection and processing in enhancement pipeline for .insertr-add elements
• Add template extraction and storage from existing HTML children with multi-variant support
• Enable collection reconstruction from database on server restart with proper DOM rebuilding
• Extend ContentClient interface with collection operations and full database integration
• Update enhance command to use engine.DatabaseClient for collection persistence support
This commit is contained in:
2025-09-22 18:29:58 +02:00
parent b25663f76b
commit 2315ba4750
36 changed files with 4356 additions and 46 deletions

View File

@@ -217,3 +217,196 @@ func toNullString(s string) sql.NullString {
}
return sql.NullString{String: s, Valid: true}
}
// GetCollection retrieves a collection container
func (c *DatabaseClient) GetCollection(siteID, collectionID string) (*CollectionItem, error) {
switch c.database.GetDBType() {
case "sqlite3":
collection, err := c.database.GetSQLiteQueries().GetCollection(context.Background(), sqlite.GetCollectionParams{
ID: collectionID,
SiteID: siteID,
})
if err != nil {
return nil, err
}
return &CollectionItem{
ID: collection.ID,
SiteID: collection.SiteID,
ContainerHTML: collection.ContainerHtml,
LastEditedBy: collection.LastEditedBy,
}, nil
case "postgresql":
collection, err := c.database.GetPostgreSQLQueries().GetCollection(context.Background(), postgresql.GetCollectionParams{
ID: collectionID,
SiteID: siteID,
})
if err != nil {
return nil, err
}
return &CollectionItem{
ID: collection.ID,
SiteID: collection.SiteID,
ContainerHTML: collection.ContainerHtml,
LastEditedBy: collection.LastEditedBy,
}, nil
default:
return nil, fmt.Errorf("unsupported database type: %s", c.database.GetDBType())
}
}
// CreateCollection creates a new collection container
func (c *DatabaseClient) CreateCollection(siteID, collectionID, containerHTML, lastEditedBy string) (*CollectionItem, error) {
switch c.database.GetDBType() {
case "sqlite3":
collection, err := c.database.GetSQLiteQueries().CreateCollection(context.Background(), sqlite.CreateCollectionParams{
ID: collectionID,
SiteID: siteID,
ContainerHtml: containerHTML,
LastEditedBy: lastEditedBy,
})
if err != nil {
return nil, err
}
return &CollectionItem{
ID: collection.ID,
SiteID: collection.SiteID,
ContainerHTML: collection.ContainerHtml,
LastEditedBy: collection.LastEditedBy,
}, nil
case "postgresql":
collection, err := c.database.GetPostgreSQLQueries().CreateCollection(context.Background(), postgresql.CreateCollectionParams{
ID: collectionID,
SiteID: siteID,
ContainerHtml: containerHTML,
LastEditedBy: lastEditedBy,
})
if err != nil {
return nil, err
}
return &CollectionItem{
ID: collection.ID,
SiteID: collection.SiteID,
ContainerHTML: collection.ContainerHtml,
LastEditedBy: collection.LastEditedBy,
}, nil
default:
return nil, fmt.Errorf("unsupported database type: %s", c.database.GetDBType())
}
}
// GetCollectionItems retrieves all items in a collection with template information
func (c *DatabaseClient) GetCollectionItems(siteID, collectionID string) ([]CollectionItemWithTemplate, error) {
switch c.database.GetDBType() {
case "sqlite3":
items, err := c.database.GetSQLiteQueries().GetCollectionItemsWithTemplate(context.Background(), sqlite.GetCollectionItemsWithTemplateParams{
CollectionID: collectionID,
SiteID: siteID,
})
if err != nil {
return nil, err
}
result := make([]CollectionItemWithTemplate, len(items))
for i, item := range items {
result[i] = CollectionItemWithTemplate{
ItemID: item.ItemID,
CollectionID: item.CollectionID,
SiteID: item.SiteID,
TemplateID: int(item.TemplateID),
HTMLContent: item.HtmlContent,
Position: int(item.Position),
LastEditedBy: item.LastEditedBy,
TemplateName: item.TemplateName,
HTMLTemplate: item.HtmlTemplate,
IsDefault: item.IsDefault != 0, // SQLite uses INTEGER for boolean
}
}
return result, nil
case "postgresql":
items, err := c.database.GetPostgreSQLQueries().GetCollectionItemsWithTemplate(context.Background(), postgresql.GetCollectionItemsWithTemplateParams{
CollectionID: collectionID,
SiteID: siteID,
})
if err != nil {
return nil, err
}
result := make([]CollectionItemWithTemplate, len(items))
for i, item := range items {
result[i] = CollectionItemWithTemplate{
ItemID: item.ItemID,
CollectionID: item.CollectionID,
SiteID: item.SiteID,
TemplateID: int(item.TemplateID),
HTMLContent: item.HtmlContent,
Position: int(item.Position),
LastEditedBy: item.LastEditedBy,
TemplateName: item.TemplateName,
HTMLTemplate: item.HtmlTemplate,
IsDefault: item.IsDefault, // PostgreSQL uses BOOLEAN
}
}
return result, nil
default:
return nil, fmt.Errorf("unsupported database type: %s", c.database.GetDBType())
}
}
// CreateCollectionTemplate creates a new template for a collection
func (c *DatabaseClient) CreateCollectionTemplate(siteID, collectionID, name, htmlTemplate string, isDefault bool) (*CollectionTemplateItem, error) {
switch c.database.GetDBType() {
case "sqlite3":
var isDefaultInt int64
if isDefault {
isDefaultInt = 1
}
template, err := c.database.GetSQLiteQueries().CreateCollectionTemplate(context.Background(), sqlite.CreateCollectionTemplateParams{
CollectionID: collectionID,
SiteID: siteID,
Name: name,
HtmlTemplate: htmlTemplate,
IsDefault: isDefaultInt,
})
if err != nil {
return nil, err
}
return &CollectionTemplateItem{
TemplateID: int(template.TemplateID),
CollectionID: template.CollectionID,
SiteID: template.SiteID,
Name: template.Name,
HTMLTemplate: template.HtmlTemplate,
IsDefault: template.IsDefault != 0,
}, nil
case "postgresql":
template, err := c.database.GetPostgreSQLQueries().CreateCollectionTemplate(context.Background(), postgresql.CreateCollectionTemplateParams{
CollectionID: collectionID,
SiteID: siteID,
Name: name,
HtmlTemplate: htmlTemplate,
IsDefault: isDefault,
})
if err != nil {
return nil, err
}
return &CollectionTemplateItem{
TemplateID: int(template.TemplateID),
CollectionID: template.CollectionID,
SiteID: template.SiteID,
Name: template.Name,
HTMLTemplate: template.HtmlTemplate,
IsDefault: template.IsDefault,
}, nil
default:
return nil, fmt.Errorf("unsupported database type: %s", c.database.GetDBType())
}
}

View File

@@ -52,14 +52,14 @@ func (e *ContentEngine) ProcessContent(input ContentInput) (*ContentResult, erro
return nil, fmt.Errorf("parsing HTML: %w", err)
}
// 2. Find insertr elements
elements := e.findInsertrElements(doc)
// 2. Find insertr and collection elements
insertrElements, collectionElements := e.findEditableElements(doc)
// 3. Generate IDs for elements
// 3. Process regular .insertr elements
generatedIDs := make(map[string]string)
processedElements := make([]ProcessedElement, len(elements))
processedElements := make([]ProcessedElement, len(insertrElements))
for i, elem := range elements {
for i, elem := range insertrElements {
// Generate structural ID (always deterministic)
id := e.idGenerator.Generate(elem.Node, input.FilePath)
@@ -97,7 +97,26 @@ func (e *ContentEngine) ProcessContent(input ContentInput) (*ContentResult, erro
}
}
// 4. Inject content if required by mode
// 4. Process .insertr-add collection elements
for _, collectionElem := range collectionElements {
// Generate structural ID for the collection container
collectionID := e.idGenerator.Generate(collectionElem.Node, input.FilePath)
// Add data-content-id attribute to the collection container
e.setAttribute(collectionElem.Node, "data-content-id", collectionID)
// Process collection during enhancement or content injection
if input.Mode == Enhancement || input.Mode == ContentInjection {
err := e.processCollection(collectionElem.Node, collectionID, input.SiteID)
if err != nil {
fmt.Printf("⚠️ Failed to process collection %s: %v\n", collectionID, err)
} else {
fmt.Printf("✅ Processed collection: %s\n", collectionID)
}
}
}
// 5. Inject content if required by mode
if input.Mode == Enhancement || input.Mode == ContentInjection {
err = e.injectContent(processedElements, input.SiteID)
if err != nil {
@@ -105,7 +124,9 @@ func (e *ContentEngine) ProcessContent(input ContentInput) (*ContentResult, erro
}
}
// 5. Inject editor assets for enhancement mode (development)
// TODO: Implement collection-specific content injection here if needed
// 6. Inject editor assets for enhancement mode (development)
if input.Mode == Enhancement {
injector := NewInjectorWithAuth(e.client, input.SiteID, e.authProvider)
injector.InjectEditorAssets(doc, true, "")
@@ -123,31 +144,40 @@ type InsertrElement struct {
Node *html.Node
}
// findInsertrElements finds all elements with class="insertr" and applies container transformation
// This implements the "syntactic sugar transformation" from CLASSES.md:
// - Containers with .insertr get their .insertr class removed
// - Viable children of those containers get .insertr class added
// - Regular elements with .insertr are kept as-is
func (e *ContentEngine) findInsertrElements(doc *html.Node) []InsertrElement {
var elements []InsertrElement
// CollectionElement represents an insertr-add collection element found in HTML
type CollectionElement struct {
Node *html.Node
}
// findEditableElements finds all editable elements (.insertr and .insertr-add)
func (e *ContentEngine) findEditableElements(doc *html.Node) ([]InsertrElement, []CollectionElement) {
var insertrElements []InsertrElement
var collectionElements []CollectionElement
var containersToTransform []*html.Node
// First pass: find all .insertr elements and identify containers
// First pass: find all .insertr and .insertr-add elements
e.walkNodes(doc, func(n *html.Node) {
if n.Type == html.ElementNode && e.hasInsertrClass(n) {
if isContainer(n) {
// Container element - mark for transformation
containersToTransform = append(containersToTransform, n)
} else {
// Regular element - add directly
elements = append(elements, InsertrElement{
if n.Type == html.ElementNode {
if e.hasInsertrClass(n) {
if isContainer(n) {
// Container element - mark for transformation
containersToTransform = append(containersToTransform, n)
} else {
// Regular element - add directly
insertrElements = append(insertrElements, InsertrElement{
Node: n,
})
}
} else if e.hasInsertrAddClass(n) {
// Collection element - add directly (no container transformation for collections)
collectionElements = append(collectionElements, CollectionElement{
Node: n,
})
}
}
})
// Second pass: transform containers (remove .insertr from container, add to children)
// Second pass: transform .insertr containers (remove .insertr from container, add to children)
for _, container := range containersToTransform {
// Remove .insertr class from container
e.removeClass(container, "insertr")
@@ -156,13 +186,23 @@ func (e *ContentEngine) findInsertrElements(doc *html.Node) []InsertrElement {
viableChildren := FindViableChildren(container)
for _, child := range viableChildren {
e.addClass(child, "insertr")
elements = append(elements, InsertrElement{
insertrElements = append(insertrElements, InsertrElement{
Node: child,
})
}
}
return elements
return insertrElements, collectionElements
}
// findInsertrElements finds all elements with class="insertr" and applies container transformation
// This implements the "syntactic sugar transformation" from CLASSES.md:
// - Containers with .insertr get their .insertr class removed
// - Viable children of those containers get .insertr class added
// - Regular elements with .insertr are kept as-is
func (e *ContentEngine) findInsertrElements(doc *html.Node) []InsertrElement {
insertrElements, _ := e.findEditableElements(doc)
return insertrElements
}
// walkNodes walks through all nodes in the document
@@ -184,6 +224,17 @@ func (e *ContentEngine) hasInsertrClass(node *html.Node) bool {
return false
}
// hasInsertrAddClass checks if node has class="insertr-add" (collection)
func (e *ContentEngine) hasInsertrAddClass(node *html.Node) bool {
classes := GetClasses(node)
for _, class := range classes {
if class == "insertr-add" {
return true
}
}
return false
}
// addContentAttributes adds data-content-id attribute only
// HTML-first approach: no content-type attribute needed
func (e *ContentEngine) addContentAttributes(node *html.Node, contentID string) {
@@ -342,3 +393,124 @@ func (e *ContentEngine) extractOriginalTemplate(node *html.Node) string {
}
return buf.String()
}
// processCollection handles collection detection, persistence and reconstruction
func (e *ContentEngine) processCollection(collectionNode *html.Node, collectionID, siteID string) error {
// 1. Check if collection exists in database
existingCollection, err := e.client.GetCollection(siteID, collectionID)
collectionExists := (err == nil && existingCollection != nil)
if !collectionExists {
// 2. New collection: extract container HTML and create collection record
containerHTML := e.extractOriginalTemplate(collectionNode)
_, err := e.client.CreateCollection(siteID, collectionID, containerHTML, "system")
if err != nil {
return fmt.Errorf("failed to create collection %s: %w", collectionID, err)
}
// 3. Extract templates from existing children
err = e.extractAndStoreTemplates(collectionNode, collectionID, siteID)
if err != nil {
return fmt.Errorf("failed to extract templates for collection %s: %w", collectionID, err)
}
fmt.Printf("✅ Created new collection: %s with templates\n", collectionID)
} else {
// 4. Existing collection: reconstruct items from database
err = e.reconstructCollectionItems(collectionNode, collectionID, siteID)
if err != nil {
return fmt.Errorf("failed to reconstruct collection %s: %w", collectionID, err)
}
fmt.Printf("✅ Reconstructed collection: %s from database\n", collectionID)
}
return nil
}
// extractAndStoreTemplates extracts template patterns from existing collection children
func (e *ContentEngine) extractAndStoreTemplates(collectionNode *html.Node, collectionID, siteID string) error {
// Find existing children elements to use as templates
var templateElements []*html.Node
// Walk through direct children of the collection
for child := collectionNode.FirstChild; child != nil; child = child.NextSibling {
if child.Type == html.ElementNode {
templateElements = append(templateElements, child)
}
}
if len(templateElements) == 0 {
// No existing children - create a default empty template
_, err := e.client.CreateCollectionTemplate(siteID, collectionID, "default", "<div>New item</div>", true)
if err != nil {
return fmt.Errorf("failed to create default template: %w", err)
}
fmt.Printf("✅ Created default template for collection %s\n", collectionID)
return nil
}
// Extract templates from existing children
for i, templateElement := range templateElements {
templateHTML := e.extractOriginalTemplate(templateElement)
templateName := fmt.Sprintf("template-%d", i+1)
isDefault := (i == 0) // First template is default
_, err := e.client.CreateCollectionTemplate(siteID, collectionID, templateName, templateHTML, isDefault)
if err != nil {
return fmt.Errorf("failed to create template %s: %w", templateName, err)
}
fmt.Printf("✅ Created template '%s' for collection %s\n", templateName, collectionID)
}
return nil
}
// reconstructCollectionItems rebuilds collection items from database and adds them to DOM
func (e *ContentEngine) reconstructCollectionItems(collectionNode *html.Node, collectionID, siteID string) error {
// Get all items for this collection from database
items, err := e.client.GetCollectionItems(siteID, collectionID)
if err != nil {
return fmt.Errorf("failed to get collection items: %w", err)
}
// Clear existing children (they will be replaced with database items)
for child := collectionNode.FirstChild; child != nil; {
next := child.NextSibling
collectionNode.RemoveChild(child)
child = next
}
// Add items from database in position order
for _, item := range items {
// Parse the item HTML content
itemDoc, err := html.Parse(strings.NewReader(item.HTMLContent))
if err != nil {
fmt.Printf("⚠️ Failed to parse item HTML for %s: %v\n", item.ItemID, err)
continue
}
// Find the body element and extract its children
var bodyNode *html.Node
e.walkNodes(itemDoc, func(n *html.Node) {
if n.Type == html.ElementNode && n.Data == "body" {
bodyNode = n
}
})
if bodyNode != nil {
// Move all children from body to collection
for bodyChild := bodyNode.FirstChild; bodyChild != nil; {
next := bodyChild.NextSibling
bodyNode.RemoveChild(bodyChild)
collectionNode.AppendChild(bodyChild)
bodyChild = next
}
}
}
fmt.Printf("✅ Reconstructed %d items for collection %s\n", len(items), collectionID)
return nil
}

View File

@@ -48,6 +48,12 @@ type ContentClient interface {
GetBulkContent(siteID string, contentIDs []string) (map[string]ContentItem, error)
GetAllContent(siteID string) (map[string]ContentItem, error)
CreateContent(siteID, contentID, htmlContent, originalTemplate, lastEditedBy string) (*ContentItem, error)
// Collection operations
GetCollection(siteID, collectionID string) (*CollectionItem, error)
CreateCollection(siteID, collectionID, containerHTML, lastEditedBy string) (*CollectionItem, error)
GetCollectionItems(siteID, collectionID string) ([]CollectionItemWithTemplate, error)
CreateCollectionTemplate(siteID, collectionID, name, htmlTemplate string, isDefault bool) (*CollectionTemplateItem, error)
}
// ContentItem represents a piece of content from the database
@@ -65,3 +71,39 @@ type ContentResponse struct {
Content []ContentItem `json:"content"`
Error string `json:"error,omitempty"`
}
// CollectionItem represents a collection container from the database
type CollectionItem struct {
ID string `json:"id"`
SiteID string `json:"site_id"`
ContainerHTML string `json:"container_html"`
UpdatedAt string `json:"updated_at"`
LastEditedBy string `json:"last_edited_by,omitempty"`
}
// CollectionTemplateItem represents a collection template from the database
type CollectionTemplateItem struct {
TemplateID int `json:"template_id"`
CollectionID string `json:"collection_id"`
SiteID string `json:"site_id"`
Name string `json:"name"`
HTMLTemplate string `json:"html_template"`
IsDefault bool `json:"is_default"`
}
// CollectionItemWithTemplate represents a collection item with its template information
type CollectionItemWithTemplate struct {
ItemID string `json:"item_id"`
CollectionID string `json:"collection_id"`
SiteID string `json:"site_id"`
TemplateID int `json:"template_id"`
HTMLContent string `json:"html_content"`
Position int `json:"position"`
UpdatedAt string `json:"updated_at"`
LastEditedBy string `json:"last_edited_by"`
// Template information
TemplateName string `json:"template_name"`
HTMLTemplate string `json:"html_template"`
IsDefault bool `json:"is_default"`
}