From 533b3d2fdde269fb44d042ab11bcde9e05e35c37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draic=20=C3=93=20Mhuiris?= Date: Tue, 9 Dec 2025 12:11:46 +0000 Subject: [PATCH 1/5] Created app install/sync/list/delete flow --- cmd/obol/main.go | 91 ++++--- internal/app/app.go | 534 ++++++++++++++++++++++++++++++++++++--- internal/app/chart.go | 61 +++++ internal/app/metadata.go | 41 +++ 4 files changed, 655 insertions(+), 72 deletions(-) create mode 100644 internal/app/chart.go create mode 100644 internal/app/metadata.go diff --git a/cmd/obol/main.go b/cmd/obol/main.go index 3214ef9..8d80fcc 100644 --- a/cmd/obol/main.go +++ b/cmd/obol/main.go @@ -6,7 +6,6 @@ import ( "os" "os/exec" "path/filepath" - "strings" "syscall" "github.com/ObolNetwork/obol-stack/internal/app" @@ -302,56 +301,87 @@ GLOBAL OPTIONS: { Name: "install", Usage: "Install a Helm chart as an application", - ArgsUsage: " [--values ]", + ArgsUsage: "", + Description: `Install a Helm chart as a managed application. + +The chart files are downloaded locally to the deployment directory, +allowing you to modify templates and values directly. + +Provide a direct HTTPS URL to a chart .tgz file. +Find chart URLs at https://artifacthub.io + +Examples: + obol app install https://charts.bitnami.com/bitnami/redis-19.0.0.tgz + obol app install https://charts.bitnami.com/bitnami/postgresql-15.0.0.tgz --name mydb --id production`, Flags: []cli.Flag{ &cli.StringFlag{ - Name: "values", - Aliases: []string{"v"}, - Usage: "Path to values override file", + Name: "name", + Usage: "Application name (defaults to chart name)", + }, + &cli.StringFlag{ + Name: "version", + Usage: "Chart version (defaults to latest)", + }, + &cli.StringFlag{ + Name: "id", + Usage: "Deployment ID (defaults to generated petname)", + }, + &cli.BoolFlag{ + Name: "force", + Aliases: []string{"f"}, + Usage: "Overwrite existing deployment", }, }, Action: func(c *cli.Context) error { if c.NArg() == 0 { - return fmt.Errorf("chart URL required (e.g., obol/ethereum or ethereum-helm-charts/ethereum-node)") + return fmt.Errorf("chart URL required\n\n" + + "Examples:\n" + + " obol app install https://charts.bitnami.com/bitnami/redis-19.0.0.tgz\n" + + " obol app install https://charts.bitnami.com/bitnami/postgresql-15.0.0.tgz\n\n" + + "Find charts at https://artifacthub.io") } - chartURL := c.Args().First() - valuesOverride := c.String("values") - // Parse chart URL: repo/chart -> repo and chart - parts := strings.SplitN(chartURL, "/", 2) - if len(parts) != 2 { - return fmt.Errorf("invalid chart URL format, use: /") + chartRef := c.Args().First() + opts := app.InstallOptions{ + Name: c.String("name"), + Version: c.String("version"), + ID: c.String("id"), + Force: c.Bool("force"), } - return app.Install(cfg, parts[1], parts[0], valuesOverride) + return app.Install(cfg, chartRef, opts) }, }, { - Name: "edit", - Usage: "Edit application helmfile or values", - ArgsUsage: "", + Name: "sync", + Usage: "Deploy application to cluster", + ArgsUsage: "/", Action: func(c *cli.Context) error { if c.NArg() == 0 { - return fmt.Errorf("application path required (e.g., obol/ethereum)") + return fmt.Errorf("deployment identifier required (e.g., postgresql/eager-fox)") } - appPath := c.Args().First() - return app.Edit(cfg, appPath) + return app.Sync(cfg, c.Args().First()) }, }, { - Name: "sync", - Usage: "Deploy application to cluster via helmfile", - ArgsUsage: "", + Name: "list", + Usage: "List installed applications", + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + Usage: "Show detailed information", + }, + }, Action: func(c *cli.Context) error { - if c.NArg() == 0 { - return fmt.Errorf("application path required (e.g., obol/ethereum)") + opts := app.ListOptions{ + Verbose: c.Bool("verbose"), } - appPath := c.Args().First() - return app.Sync(cfg, appPath) + return app.List(cfg, opts) }, }, { Name: "delete", - Usage: "Remove application and clean up cluster resources", - ArgsUsage: "", + Usage: "Remove application and cluster resources", + ArgsUsage: "/", Flags: []cli.Flag{ &cli.BoolFlag{ Name: "force", @@ -361,10 +391,9 @@ GLOBAL OPTIONS: }, Action: func(c *cli.Context) error { if c.NArg() == 0 { - return fmt.Errorf("application path required (e.g., obol/ethereum)") + return fmt.Errorf("deployment identifier required (e.g., postgresql/eager-fox)") } - appPath := c.Args().First() - return app.Delete(cfg, appPath, c.Bool("force")) + return app.Delete(cfg, c.Args().First(), c.Bool("force")) }, }, }, diff --git a/internal/app/app.go b/internal/app/app.go index 81eef35..0980174 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -1,64 +1,516 @@ package app import ( + "bytes" "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "text/template" + "time" "github.com/ObolNetwork/obol-stack/internal/config" + "github.com/dustinkirkland/golang-petname" + "gopkg.in/yaml.v3" ) -// TODO: Application Installation System -// -// The applications system is being refactored to use a helmfile-based composition pattern. -// -// Current architecture: -// - Root helmfile.yaml: $OBOL_CONFIG_DIR/helmfile.yaml -// - Per-app helmfiles: $OBOL_CONFIG_DIR/applications/{repo}/{chart}/helmfile.yaml -// - Per-app values: $OBOL_CONFIG_DIR/applications/{repo}/{chart}/values.yaml -// -// Implementation needed: -// 1. Install(cfg, chart, repo, valuesOverride) - Scaffold application directory -// 2. Edit(cfg, appPath) - Open helmfile or values.yaml in editor -// 3. Sync(cfg, appPath) - Deploy via helmfile -// 4. Delete(cfg, appPath) - Remove app and clean up cluster resources -// -// See: internal/embed/helmfile.yaml for root orchestration pattern - -// Install scaffolds a new application directory -func Install(cfg *config.Config, chart string, repo string, valuesOverride string) error { - fmt.Printf("Installing application: %s/%s\n", repo, chart) - fmt.Println("TODO: Implement application scaffolding") - fmt.Println(" 1. Validate chart exists in repo") - fmt.Println(" 2. Create: $OBOL_CONFIG_DIR/applications/{repo}/{chart}/") - fmt.Println(" 3. Generate helmfile.yaml referencing chart") - fmt.Println(" 4. Generate values.yaml with sane defaults") +// InstallOptions contains options for the install command +type InstallOptions struct { + Name string // Optional app name override + Version string // Chart version (empty = latest) + ID string // Deployment ID (empty = generate petname) + Force bool // Overwrite existing deployment +} + +// ListOptions contains options for the list command +type ListOptions struct { + Verbose bool // Show detailed information +} + +// Install scaffolds a new application from a Helm chart +func Install(cfg *config.Config, chartRef string, opts InstallOptions) error { + fmt.Printf("Installing application from: %s\n", chartRef) + + // 1. Parse chart reference + chart, err := ParseChartReference(chartRef) + if err != nil { + return err + } + + // 2. Determine app name + appName := opts.Name + if appName == "" { + appName = chart.GetChartName() + } + fmt.Printf("Application name: %s\n", appName) + + // 3. Generate or use provided ID + id := opts.ID + if id == "" { + id = petname.Generate(2, "-") + fmt.Printf("Generated deployment ID: %s\n", id) + } else { + fmt.Printf("Using deployment ID: %s\n", id) + } + + // 4. Check if deployment exists + deploymentDir := filepath.Join(cfg.ConfigDir, "applications", appName, id) + if _, err := os.Stat(deploymentDir); err == nil { + if !opts.Force { + return fmt.Errorf("deployment already exists: %s/%s\n"+ + "Directory: %s\n"+ + "Use --force or -f to overwrite", appName, id, deploymentDir) + } + fmt.Printf("⚠️ WARNING: Overwriting existing deployment at %s\n", deploymentDir) + } + + // 5. Create deployment directory + if err := os.MkdirAll(deploymentDir, 0755); err != nil { + return fmt.Errorf("failed to create deployment directory: %w", err) + } + + // 6. Pull chart to local directory + version := opts.Version + fetchedVersion, err := pullChart(cfg, chart, deploymentDir, version) + if err != nil { + return fmt.Errorf("failed to pull chart: %w", err) + } + + if version == "" { + version = fetchedVersion + } + fmt.Printf("Chart version: %s\n", version) + + // 7. Get default values from the pulled chart + values, err := getChartDefaultValues(deploymentDir) + if err != nil { + return fmt.Errorf("failed to get chart values: %w", err) + } + + // 8. Write values.yaml + valuesPath := filepath.Join(deploymentDir, "values.yaml") + if err := os.WriteFile(valuesPath, values, 0644); err != nil { + return fmt.Errorf("failed to write values.yaml: %w", err) + } + + // 9. Generate helmfile.yaml (references local chart) + if err := generateHelmfile(deploymentDir, chart, appName, id, version); err != nil { + return fmt.Errorf("failed to generate helmfile: %w", err) + } + + // 10. Save metadata + meta := &Metadata{ + ChartURL: chart.ChartURL, + ChartName: chart.ChartName, + Version: version, + InstalledAt: time.Now(), + UpdatedAt: time.Now(), + } + if err := SaveMetadata(deploymentDir, meta); err != nil { + return fmt.Errorf("failed to save metadata: %w", err) + } + + // 11. Print success message + fmt.Printf("\n✓ Application installed successfully!\n") + fmt.Printf("Deployment: %s/%s\n", appName, id) + fmt.Printf("Location: %s\n", deploymentDir) + fmt.Printf("\nChart files downloaded locally:\n") + fmt.Printf(" - Chart.yaml: Chart metadata\n") + fmt.Printf(" - values.yaml: Chart default values (edit to customize)\n") + fmt.Printf(" - templates/: Kubernetes resource templates\n") + fmt.Printf(" - helmfile.yaml: Deployment definition\n") + fmt.Printf(" - metadata.yaml: Installation metadata\n") + fmt.Printf("\nYou can modify the chart templates and values directly.\n") + fmt.Printf("To deploy, run: obol app sync %s/%s\n", appName, id) return nil } -// Edit opens an application file in the user's editor -func Edit(cfg *config.Config, appPath string) error { - fmt.Printf("Editing application: %s\n", appPath) - fmt.Println("TODO: Implement editor integration") +// pullChart downloads the chart files to the deployment directory +func pullChart(cfg *config.Config, chart *ChartReference, destDir, version string) (string, error) { + helmPath := filepath.Join(cfg.BinDir, "helm") + + fmt.Printf("Pulling chart: %s\n", chart.ChartURL) + + // Pull chart with helm pull --untar + args := []string{"pull", chart.ChartURL, "--untar", "--untardir", destDir} + if version != "" { + args = append(args, "--version", version) + } + + cmd := exec.Command(helmPath, args...) + var stderr bytes.Buffer + cmd.Stdout = os.Stdout + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("helm pull failed: %w\n%s", err, stderr.String()) + } + + // Find the extracted chart directory (helm pull creates a subdirectory with chart name) + entries, err := os.ReadDir(destDir) + if err != nil { + return "", fmt.Errorf("failed to read dest directory: %w", err) + } + + var chartDir string + for _, entry := range entries { + if entry.IsDir() { + chartDir = filepath.Join(destDir, entry.Name()) + break + } + } + + if chartDir == "" { + return "", fmt.Errorf("no chart directory found after pull") + } + + // Move chart contents up one level (flatten directory structure) + // Note: Dependencies will be resolved by helmfile during sync + chartEntries, err := os.ReadDir(chartDir) + if err != nil { + return "", fmt.Errorf("failed to read chart directory: %w", err) + } + + for _, entry := range chartEntries { + src := filepath.Join(chartDir, entry.Name()) + dst := filepath.Join(destDir, entry.Name()) + if err := os.Rename(src, dst); err != nil { + return "", fmt.Errorf("failed to move %s: %w", entry.Name(), err) + } + } + + // Remove the now-empty chart directory + os.Remove(chartDir) + + // Read version from Chart.yaml + chartYamlPath := filepath.Join(destDir, "Chart.yaml") + chartYamlData, err := os.ReadFile(chartYamlPath) + if err != nil { + return version, nil // Return requested version if can't read Chart.yaml + } + + var chartInfo struct { + Version string `yaml:"version"` + } + if err := yaml.Unmarshal(chartYamlData, &chartInfo); err == nil { + return chartInfo.Version, nil + } + + return version, nil +} + +// getChartDefaultValues reads the default values from the local chart +func getChartDefaultValues(chartDir string) ([]byte, error) { + valuesPath := filepath.Join(chartDir, "values.yaml") + data, err := os.ReadFile(valuesPath) + if err != nil { + // Chart might not have a values.yaml, that's ok + if os.IsNotExist(err) { + return []byte{}, nil + } + return nil, fmt.Errorf("failed to read values.yaml: %w", err) + } + return data, nil +} + +// generateHelmfile creates the helmfile.yaml for the app +func generateHelmfile(dir string, chart *ChartReference, appName, id, version string) error { + tmpl := `# Generated by obol app install +# Chart: {{ .ChartURL }} +# Chart files are stored locally in this directory + +releases: + - name: {{ .AppName }} + namespace: {{ .Namespace }} + createNamespace: true + chart: . + values: + - values.yaml +` + + data := map[string]interface{}{ + "ChartURL": chart.ChartURL, + "AppName": appName, + "Namespace": fmt.Sprintf("%s-%s", appName, id), + } + + t, err := template.New("helmfile").Parse(tmpl) + if err != nil { + return err + } + + var buf bytes.Buffer + if err := t.Execute(&buf, data); err != nil { + return err + } + + return os.WriteFile(filepath.Join(dir, "helmfile.yaml"), buf.Bytes(), 0644) +} + +// Sync deploys or updates an application to the cluster +func Sync(cfg *config.Config, deploymentIdentifier string) error { + // Parse deployment identifier: app-name/id + appName, id, err := parseDeploymentIdentifier(deploymentIdentifier) + if err != nil { + return err + } + + fmt.Printf("Syncing application: %s/%s\n", appName, id) + + // Locate deployment directory + deploymentDir := filepath.Join(cfg.ConfigDir, "applications", appName, id) + if _, err := os.Stat(deploymentDir); os.IsNotExist(err) { + return fmt.Errorf("deployment not found: %s\nDirectory: %s", deploymentIdentifier, deploymentDir) + } + + // Check required files exist + helmfilePath := filepath.Join(deploymentDir, "helmfile.yaml") + if _, err := os.Stat(helmfilePath); os.IsNotExist(err) { + return fmt.Errorf("helmfile.yaml not found in: %s", deploymentDir) + } + + valuesPath := filepath.Join(deploymentDir, "values.yaml") + if _, err := os.Stat(valuesPath); os.IsNotExist(err) { + return fmt.Errorf("values.yaml not found in: %s", deploymentDir) + } + + // Check kubeconfig exists (cluster must be running) + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + if _, err := os.Stat(kubeconfigPath); os.IsNotExist(err) { + return fmt.Errorf("cluster not running. Run 'obol stack up' first") + } + + // Get helmfile binary path + helmfileBinary := filepath.Join(cfg.BinDir, "helmfile") + if _, err := os.Stat(helmfileBinary); os.IsNotExist(err) { + return fmt.Errorf("helmfile not found at %s", helmfileBinary) + } + + fmt.Printf("Deployment directory: %s\n", deploymentDir) + fmt.Printf("Deployment ID: %s\n", id) + fmt.Printf("Running helmfile sync...\n\n") + + // Execute helmfile sync + cmd := exec.Command(helmfileBinary, "-f", helmfilePath, "sync") + cmd.Dir = deploymentDir + cmd.Env = append(os.Environ(), + fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath), + ) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return fmt.Errorf("helmfile sync failed: %w", err) + } + + // Update metadata timestamp + if meta, err := LoadMetadata(deploymentDir); err == nil { + meta.UpdatedAt = time.Now() + SaveMetadata(deploymentDir, meta) + } + + namespace := fmt.Sprintf("%s-%s", appName, id) + fmt.Printf("\n✓ Application synced successfully!\n") + fmt.Printf("Namespace: %s\n", namespace) + fmt.Printf("\nTo check status: obol kubectl get all -n %s\n", namespace) + fmt.Printf("To view logs: obol kubectl logs -n %s \n", namespace) return nil } -// Sync deploys the application using helmfile -func Sync(cfg *config.Config, appPath string) error { - fmt.Printf("Syncing application: %s\n", appPath) - fmt.Println("TODO: Implement helmfile sync") - fmt.Println(" 1. Run: helmfile -f $OBOL_CONFIG_DIR/helmfile.yaml sync") - fmt.Println(" 2. Or: helmfile -f {appPath}/helmfile.yaml sync") +// parseDeploymentIdentifier parses "app-name/id" format +func parseDeploymentIdentifier(identifier string) (appName, id string, err error) { + // Try slash separator + if strings.Contains(identifier, "/") { + parts := strings.SplitN(identifier, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", fmt.Errorf("invalid format. Use: /") + } + return parts[0], parts[1], nil + } + + return "", "", fmt.Errorf("please use / format (e.g., postgresql/eager-fox)") +} + +// List displays installed applications +func List(cfg *config.Config, opts ListOptions) error { + appsDir := filepath.Join(cfg.ConfigDir, "applications") + + // Check if applications directory exists + if _, err := os.Stat(appsDir); os.IsNotExist(err) { + fmt.Println("No applications installed") + fmt.Println("\nTo install an application, provide a chart URL:") + fmt.Println(" obol app install https://charts.bitnami.com/bitnami/redis-19.0.0.tgz") + fmt.Println(" obol app install https://charts.bitnami.com/bitnami/postgresql-15.0.0.tgz") + fmt.Println("\nFind charts at https://artifacthub.io") + return nil + } + + // Walk through applications directory + apps, err := os.ReadDir(appsDir) + if err != nil { + return fmt.Errorf("failed to read applications directory: %w", err) + } + + if len(apps) == 0 { + fmt.Println("No applications installed") + return nil + } + + fmt.Println("Installed applications:") + fmt.Println() + + count := 0 + for _, appDir := range apps { + if !appDir.IsDir() { + continue + } + + appName := appDir.Name() + appPath := filepath.Join(appsDir, appName) + + // List deployments for this app + deployments, err := os.ReadDir(appPath) + if err != nil { + continue + } + + for _, deployment := range deployments { + if !deployment.IsDir() { + continue + } + + id := deployment.Name() + deploymentPath := filepath.Join(appPath, id) + + // Load metadata + meta, err := LoadMetadata(deploymentPath) + if err != nil { + // Metadata not found - show basic info + fmt.Printf(" %s/%s\n", appName, id) + count++ + continue + } + + // Show deployment info + if opts.Verbose { + fmt.Printf(" %s/%s\n", appName, id) + fmt.Printf(" Chart: %s\n", meta.ChartURL) + fmt.Printf(" Version: %s\n", meta.Version) + fmt.Printf(" Installed: %s\n", meta.InstalledAt.Format("2006-01-02 15:04:05")) + if !meta.UpdatedAt.IsZero() && meta.UpdatedAt != meta.InstalledAt { + fmt.Printf(" Updated: %s\n", meta.UpdatedAt.Format("2006-01-02 15:04:05")) + } + fmt.Println() + } else { + fmt.Printf(" %s/%s (chart: %s, version: %s)\n", + appName, id, meta.ChartURL, meta.Version) + } + count++ + } + } + + fmt.Printf("\nTotal: %d application deployment(s)\n", count) return nil } -// Delete removes the application and cluster resources -func Delete(cfg *config.Config, appPath string, force bool) error { - fmt.Printf("Deleting application: %s\n", appPath) - fmt.Println("TODO: Implement application deletion") - fmt.Println(" 1. Remove $OBOL_CONFIG_DIR/applications/{repo}/{chart}/") - fmt.Println(" 2. Run: kubectl delete namespace {chart}") +// Delete removes an application deployment and its cluster resources +func Delete(cfg *config.Config, deploymentIdentifier string, force bool) error { + appName, id, err := parseDeploymentIdentifier(deploymentIdentifier) + if err != nil { + return err + } + + namespaceName := fmt.Sprintf("%s-%s", appName, id) + deploymentDir := filepath.Join(cfg.ConfigDir, "applications", appName, id) + + fmt.Printf("Deleting application: %s/%s\n", appName, id) + fmt.Printf("Namespace: %s\n", namespaceName) + fmt.Printf("Config directory: %s\n", deploymentDir) + + // Check if config directory exists + configExists := false + if _, err := os.Stat(deploymentDir); err == nil { + configExists = true + } + + // Check if namespace exists in cluster + namespaceExists := false + kubeconfigPath := filepath.Join(cfg.ConfigDir, "kubeconfig.yaml") + if _, err := os.Stat(kubeconfigPath); err == nil { + kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") + cmd := exec.Command(kubectlBinary, "get", "namespace", namespaceName) + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath)) + if err := cmd.Run(); err == nil { + namespaceExists = true + } + } + + // Display what will be deleted + fmt.Println("\nResources to be deleted:") + if namespaceExists { + fmt.Printf(" [x] Kubernetes namespace: %s\n", namespaceName) + } else { + fmt.Printf(" [ ] Kubernetes namespace: %s (not found)\n", namespaceName) + } + if configExists { + fmt.Printf(" [x] Configuration directory: %s\n", deploymentDir) + } else { + fmt.Printf(" [ ] Configuration directory: %s (not found)\n", deploymentDir) + } + + // Check if there's anything to delete + if !namespaceExists && !configExists { + return fmt.Errorf("deployment not found: %s", deploymentIdentifier) + } + + // Confirm deletion (unless --force) + if !force { + fmt.Print("\nProceed with deletion? [y/N]: ") + var response string + fmt.Scanln(&response) + if strings.ToLower(response) != "y" && strings.ToLower(response) != "yes" { + fmt.Println("Deletion cancelled") + return nil + } + } + + // Delete Kubernetes namespace + if namespaceExists { + fmt.Printf("\nDeleting namespace %s...\n", namespaceName) + kubectlBinary := filepath.Join(cfg.BinDir, "kubectl") + cmd := exec.Command(kubectlBinary, "delete", "namespace", namespaceName, + "--force", "--grace-period=0") + cmd.Env = append(os.Environ(), fmt.Sprintf("KUBECONFIG=%s", kubeconfigPath)) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + return fmt.Errorf("failed to delete namespace: %w", err) + } + fmt.Println("Namespace deleted") + } + + // Delete configuration directory + if configExists { + fmt.Printf("Deleting configuration directory...\n") + if err := os.RemoveAll(deploymentDir); err != nil { + return fmt.Errorf("failed to delete config directory: %w", err) + } + fmt.Println("Configuration deleted") + + // Clean up empty parent directories + appDir := filepath.Join(cfg.ConfigDir, "applications", appName) + entries, err := os.ReadDir(appDir) + if err == nil && len(entries) == 0 { + os.Remove(appDir) + } + } + + fmt.Printf("\n✓ Application %s/%s deleted successfully!\n", appName, id) return nil } diff --git a/internal/app/chart.go b/internal/app/chart.go new file mode 100644 index 0000000..054e81f --- /dev/null +++ b/internal/app/chart.go @@ -0,0 +1,61 @@ +package app + +import ( + "fmt" + "net/url" + "path/filepath" + "regexp" + "strings" +) + +// ChartReference holds parsed chart information +type ChartReference struct { + Original string // Original input string + ChartName string // Chart name extracted from URL + ChartURL string // Full URL to chart +} + +// ParseChartReference parses a chart URL +func ParseChartReference(ref string) (*ChartReference, error) { + ref = strings.TrimSpace(ref) + + // Only support HTTPS/HTTP URLs + if !strings.HasPrefix(ref, "https://") && !strings.HasPrefix(ref, "http://") { + return nil, fmt.Errorf("invalid chart URL: %s\n"+ + "Please provide a direct HTTPS URL to a chart .tgz file\n"+ + "Example: https://charts.bitnami.com/bitnami/redis-19.0.0.tgz\n"+ + "Find chart URLs at https://artifacthub.io", ref) + } + + return parseURLReference(ref) +} + +func parseURLReference(ref string) (*ChartReference, error) { + u, err := url.Parse(ref) + if err != nil { + return nil, fmt.Errorf("invalid URL: %w", err) + } + + // Extract chart name from URL path (e.g., redis-19.0.0.tgz -> redis) + path := u.Path + base := filepath.Base(path) + + // Remove .tgz extension + chartName := strings.TrimSuffix(base, ".tgz") + // Remove version suffix (e.g., redis-19.0.0 -> redis) + re := regexp.MustCompile(`^(.+)-\d+\.\d+\.\d+.*$`) + if matches := re.FindStringSubmatch(chartName); len(matches) > 1 { + chartName = matches[1] + } + + return &ChartReference{ + Original: ref, + ChartName: chartName, + ChartURL: ref, + }, nil +} + +// GetChartName returns the name to use for the app directory +func (c *ChartReference) GetChartName() string { + return c.ChartName +} diff --git a/internal/app/metadata.go b/internal/app/metadata.go new file mode 100644 index 0000000..63ad435 --- /dev/null +++ b/internal/app/metadata.go @@ -0,0 +1,41 @@ +package app + +import ( + "os" + "path/filepath" + "time" + + "gopkg.in/yaml.v3" +) + +// Metadata stores information about an installed application +type Metadata struct { + ChartURL string `yaml:"chartUrl"` // Chart download URL + ChartName string `yaml:"chartName"` // Extracted chart name + Version string `yaml:"version"` // Chart version + InstalledAt time.Time `yaml:"installedAt"` // Installation timestamp + UpdatedAt time.Time `yaml:"updatedAt,omitempty"` // Last update timestamp +} + +// SaveMetadata writes metadata to the deployment directory +func SaveMetadata(dir string, meta *Metadata) error { + data, err := yaml.Marshal(meta) + if err != nil { + return err + } + return os.WriteFile(filepath.Join(dir, "metadata.yaml"), data, 0644) +} + +// LoadMetadata reads metadata from a deployment directory +func LoadMetadata(dir string) (*Metadata, error) { + data, err := os.ReadFile(filepath.Join(dir, "metadata.yaml")) + if err != nil { + return nil, err + } + + var meta Metadata + if err := yaml.Unmarshal(data, &meta); err != nil { + return nil, err + } + return &meta, nil +} From 4eb53507a45396a1ba01d90e31163a45b8fb2768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draic=20=C3=93=20Mhuiris?= Date: Tue, 9 Dec 2025 12:22:03 +0000 Subject: [PATCH 2/5] updated readme --- README.md | 188 ++++++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 167 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 7bc5d4a..e34b6a4 100644 --- a/README.md +++ b/README.md @@ -109,10 +109,16 @@ obol stack up obol network install ethereum # This creates a deployment like: ethereum-nervous-otter +# Deploy the network to the cluster +obol network sync ethereum/nervous-otter + # Install another network configuration obol network install ethereum --network=holesky # This creates a separate deployment like: ethereum-happy-panda +# Deploy the second network +obol network sync ethereum/happy-panda + # View cluster resources (opens interactive terminal UI) obol k9s ``` @@ -120,7 +126,10 @@ obol k9s The stack will create a local Kubernetes cluster. Each network installation creates a uniquely-namespaced deployment instance, allowing you to run multiple configurations simultaneously. > [!TIP] -> Use `obol network list` to see all available networks. Customize installations with flags (e.g., `obol network install ethereum --network=holesky --execution-client=geth`) to create different deployment configurations. +> Use `obol network list` to see all available networks. Customize installations with flags (e.g., `obol network install ethereum --network=holesky --execution-client=geth`) to create different deployment configurations. After installation, deploy to the cluster with `obol network sync /`. + +> [!TIP] +> You can also install arbitrary Helm charts as applications using `obol app install `. Find charts at [Artifact Hub](https://artifacthub.io). ## Managing Networks @@ -159,9 +168,26 @@ obol network install ethereum Each network installation creates a **unique deployment instance** with: 1. Network configuration templated and saved to `~/.config/obol/networks/ethereum//` -2. Resources deployed to a unique Kubernetes namespace (e.g., `ethereum-nervous-otter`) +2. Configuration files ready to be deployed to the cluster 3. Isolated persistent storage for blockchain data +### Sync a Network + +After installing a network configuration, deploy it to the cluster: + +```bash +# Deploy the network to the cluster +obol network sync ethereum/ + +# Or use the namespace format +obol network sync ethereum-nervous-otter +``` + +The sync command: +- Reads the configuration from `~/.config/obol/networks///` +- Executes helmfile to deploy resources to a unique Kubernetes namespace +- Creates isolated persistent storage for blockchain data + **Multiple deployments:** You can install the same network type multiple times with different configurations. Each deployment is isolated in its own namespace: @@ -169,15 +195,24 @@ You can install the same network type multiple times with different configuratio ```bash # Install mainnet with Geth + Prysm obol network install ethereum --network=mainnet --execution-client=geth --consensus-client=prysm -# Creates: ethereum-nervous-otter namespace +# Creates configuration: ethereum-nervous-otter + +# Deploy to cluster +obol network sync ethereum/nervous-otter # Install Holesky testnet with Reth + Lighthouse obol network install ethereum --network=holesky --execution-client=reth --consensus-client=lighthouse -# Creates: ethereum-laughing-elephant namespace +# Creates configuration: ethereum-laughing-elephant + +# Deploy Holesky to cluster +obol network sync ethereum/laughing-elephant # Install another Holesky instance for testing obol network install ethereum --network=holesky -# Creates: ethereum-happy-panda namespace +# Creates configuration: ethereum-happy-panda + +# Deploy second Holesky instance +obol network sync ethereum/happy-panda ``` **Ethereum configuration options:** @@ -203,10 +238,11 @@ Each network installation creates an isolated deployment with a unique namespace - **Persistent volumes**: Blockchain data stored in `~/.local/share/obol//networks/_/` - **Service endpoints**: Internal cluster DNS per deployment (e.g., `ethereum-rpc.ethereum-nervous-otter.svc.cluster.local`) -**Two-stage templating:** -1. CLI flags template the helmfile values section with your configuration -2. Helmfile processes the template and deploys to the cluster -3. Configuration is saved locally for future updates and management +**Install and Sync workflow:** +1. **Install**: `obol network install` generates configuration files locally from CLI flags +2. **Customize**: Edit values and templates in `~/.config/obol/networks///` (optional) +3. **Sync**: `obol network sync` deploys the configuration to the cluster using helmfile +4. **Update**: Modify configuration and re-sync to update the deployment ### Delete a Network Deployment @@ -234,6 +270,103 @@ This command will: > [!NOTE] > You can have multiple deployments of the same network type. Deleting one deployment (e.g., `ethereum-nervous-otter`) does not affect other deployments (e.g., `ethereum-happy-panda`). +## Managing Applications + +The Obol Stack supports installing arbitrary Helm charts as managed applications. Each application installation creates an isolated deployment with its own namespace, similar to network deployments. + +### Install an Application + +Install any Helm chart by providing a direct HTTPS URL to a chart `.tgz` file: + +```bash +# Install Redis from Bitnami +obol app install https://charts.bitnami.com/bitnami/redis-19.0.0.tgz + +# Install PostgreSQL with custom name and ID +obol app install https://charts.bitnami.com/bitnami/postgresql-15.0.0.tgz --name mydb --id production +``` + +Find chart URLs at [Artifact Hub](https://artifacthub.io). + +**What happens during installation:** +1. Downloads the chart files locally to `~/.config/obol/applications///` +2. Extracts chart templates, values, and metadata +3. Generates a helmfile.yaml for deployment +4. Saves installation metadata + +**Installation options:** +- `--name`: Application name (defaults to chart name) +- `--version`: Chart version (defaults to latest) +- `--id`: Deployment ID (defaults to generated petname) +- `--force` or `-f`: Overwrite existing deployment + +### Sync an Application + +After installing an application, deploy it to the cluster: + +```bash +# Deploy the application +obol app sync postgresql/eager-fox + +# Check status +obol kubectl get all -n postgresql-eager-fox +``` + +The sync command: +- Reads configuration from `~/.config/obol/applications///` +- Executes helmfile to deploy resources +- Creates unique namespace: `-` + +### List Applications + +View all installed applications: + +```bash +# Simple list +obol app list + +# Detailed information +obol app list --verbose +``` + +### Delete an Application + +Remove an application deployment and its cluster resources: + +```bash +# Delete with confirmation prompt +obol app delete postgresql/eager-fox + +# Skip confirmation +obol app delete postgresql/eager-fox --force +``` + +This command will: +- Remove the Kubernetes namespace and all deployed resources +- Delete the configuration directory and chart files + +### Customize Applications + +After installation, you can modify the chart files directly: + +```bash +# Edit application values +$EDITOR ~/.config/obol/applications/postgresql/eager-fox/values.yaml + +# Modify templates +$EDITOR ~/.config/obol/applications/postgresql/eager-fox/templates/statefulset.yaml + +# Re-deploy with changes +obol app sync postgresql/eager-fox +``` + +**Local files:** +- `Chart.yaml`: Chart metadata +- `values.yaml`: Configuration values (edit to customize) +- `templates/`: Kubernetes resource templates +- `helmfile.yaml`: Deployment definition +- `metadata.yaml`: Installation metadata + ### Managing the Stack **Start the stack:** @@ -354,12 +487,22 @@ The Obol Stack follows the [XDG Base Directory](https://specifications.freedeskt │ ├── helmfile.yaml # Base stack configuration │ ├── base/ # Base Kubernetes resources │ └── values/ # Configuration templates (ERPC, frontend) -└── networks/ # Installed network deployments - ├── ethereum/ # Ethereum network deployments - │ ├── / # First deployment instance - │ └── / # Second deployment instance - ├── helios/ # Helios network deployments - └── aztec/ # Aztec network deployments +├── networks/ # Installed network deployments +│ ├── ethereum/ # Ethereum network deployments +│ │ ├── / # First deployment instance +│ │ └── / # Second deployment instance +│ ├── helios/ # Helios network deployments +│ └── aztec/ # Aztec network deployments +└── applications/ # Installed application deployments + ├── redis/ # Redis deployments + │ └── / # Deployment instance + │ ├── Chart.yaml # Chart metadata + │ ├── values.yaml # Configuration values + │ ├── helmfile.yaml # Deployment definition + │ ├── metadata.yaml # Installation metadata + │ └── templates/ # Kubernetes resources + └── postgresql/ # PostgreSQL deployments + └── / # Deployment instance ``` **Data directory structure:** @@ -453,12 +596,15 @@ If you're contributing to the Obol Stack or want to run it from source, you can │ │ ├── helmfile.yaml │ │ ├── base/ │ │ └── values/ -│ └── networks/ # Installed network deployments -│ ├── ethereum/ # Ethereum network deployments -│ │ ├── / # First deployment instance -│ │ └── / # Second deployment instance -│ ├── helios/ -│ └── aztec/ +│ ├── networks/ # Installed network deployments +│ │ ├── ethereum/ # Ethereum network deployments +│ │ │ ├── / # First deployment instance +│ │ │ └── / # Second deployment instance +│ │ ├── helios/ +│ │ └── aztec/ +│ └── applications/ # Installed application deployments +│ ├── redis/ +│ └── postgresql/ └── data/ # Persistent volumes (network data) ``` From c5ceeb43c4f20a3f4227e27be78ba46664ae1ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draic=20=C3=93=20Mhuiris?= Date: Wed, 10 Dec 2025 22:02:56 +0000 Subject: [PATCH 3/5] added applications to obol splash screen --- cmd/obol/main.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/obol/main.go b/cmd/obol/main.go index 8d80fcc..8e27a84 100644 --- a/cmd/obol/main.go +++ b/cmd/obol/main.go @@ -49,6 +49,12 @@ COMMANDS: network install Install and deploy network to cluster network delete Remove network and clean up cluster resources + App Management: + app install Install a Helm chart as an application + app list List installed applications + app sync Deploy application to cluster + app delete Remove application and cluster resources + Kubernetes Tools (with auto-configured KUBECONFIG): kubectl Run kubectl with stack kubeconfig (passthrough) helm Run helm with stack kubeconfig (passthrough) From 51a6b2b4419fbd6f53b65f463b856392a11960cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draic=20=C3=93=20Mhuiris?= Date: Wed, 10 Dec 2025 22:15:12 +0000 Subject: [PATCH 4/5] using helmfile remote urls instead of local charts and enabled artifacthub chart ref resolution --- cmd/obol/main.go | 24 ++-- internal/app/app.go | 265 +++++++++++++++++------------------- internal/app/artifacthub.go | 134 ++++++++++++++++++ internal/app/chart.go | 125 ++++++++++++++--- internal/app/metadata.go | 75 ++++++---- 5 files changed, 436 insertions(+), 187 deletions(-) create mode 100644 internal/app/artifacthub.go diff --git a/cmd/obol/main.go b/cmd/obol/main.go index 8e27a84..3bd70f2 100644 --- a/cmd/obol/main.go +++ b/cmd/obol/main.go @@ -307,18 +307,22 @@ GLOBAL OPTIONS: { Name: "install", Usage: "Install a Helm chart as an application", - ArgsUsage: "", + ArgsUsage: "", Description: `Install a Helm chart as a managed application. -The chart files are downloaded locally to the deployment directory, -allowing you to modify templates and values directly. - -Provide a direct HTTPS URL to a chart .tgz file. -Find chart URLs at https://artifacthub.io +Supported chart reference formats: + repo/chart Resolved via ArtifactHub (e.g., bitnami/redis) + repo/chart@version Specific version (e.g., bitnami/redis@19.0.0) + https://.../*.tgz Direct URL to chart archive + oci://... OCI registry reference Examples: + obol app install bitnami/redis + obol app install bitnami/postgresql@15.0.0 obol app install https://charts.bitnami.com/bitnami/redis-19.0.0.tgz - obol app install https://charts.bitnami.com/bitnami/postgresql-15.0.0.tgz --name mydb --id production`, + obol app install oci://registry-1.docker.io/bitnamicharts/redis --name mydb --id production + +Find charts at https://artifacthub.io`, Flags: []cli.Flag{ &cli.StringFlag{ Name: "name", @@ -340,10 +344,12 @@ Examples: }, Action: func(c *cli.Context) error { if c.NArg() == 0 { - return fmt.Errorf("chart URL required\n\n" + + return fmt.Errorf("chart reference required\n\n" + "Examples:\n" + + " obol app install bitnami/redis\n" + + " obol app install bitnami/postgresql@15.0.0\n" + " obol app install https://charts.bitnami.com/bitnami/redis-19.0.0.tgz\n" + - " obol app install https://charts.bitnami.com/bitnami/postgresql-15.0.0.tgz\n\n" + + " obol app install oci://registry-1.docker.io/bitnamicharts/redis\n\n" + "Find charts at https://artifacthub.io") } chartRef := c.Args().First() diff --git a/internal/app/app.go b/internal/app/app.go index 0980174..33e9d5f 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -8,17 +8,15 @@ import ( "path/filepath" "strings" "text/template" - "time" "github.com/ObolNetwork/obol-stack/internal/config" "github.com/dustinkirkland/golang-petname" - "gopkg.in/yaml.v3" ) // InstallOptions contains options for the install command type InstallOptions struct { Name string // Optional app name override - Version string // Chart version (empty = latest) + Version string // Chart version (empty = latest for repo/chart, extracted for URL) ID string // Deployment ID (empty = generate petname) Force bool // Overwrite existing deployment } @@ -28,7 +26,7 @@ type ListOptions struct { Verbose bool // Show detailed information } -// Install scaffolds a new application from a Helm chart +// Install scaffolds a new application from a Helm chart reference func Install(cfg *config.Config, chartRef string, opts InstallOptions) error { fmt.Printf("Installing application from: %s\n", chartRef) @@ -38,14 +36,36 @@ func Install(cfg *config.Config, chartRef string, opts InstallOptions) error { return err } - // 2. Determine app name + // 2. If repo/chart format, resolve via ArtifactHub + if chart.NeedsResolution() { + fmt.Printf("Resolving chart via ArtifactHub...\n") + client := NewArtifactHubClient() + info, err := client.ResolveChart(chartRef) + if err != nil { + return err + } + chart.RepoURL = info.RepoURL + chart.RepoName = info.RepoName + if chart.Version == "" { + chart.Version = info.Version + } + fmt.Printf("Resolved: %s/%s version %s\n", info.RepoName, info.ChartName, info.Version) + fmt.Printf("Repository URL: %s\n", info.RepoURL) + } + + // Apply version override from CLI flag + if opts.Version != "" { + chart.Version = opts.Version + } + + // 3. Determine app name appName := opts.Name if appName == "" { appName = chart.GetChartName() } fmt.Printf("Application name: %s\n", appName) - // 3. Generate or use provided ID + // 4. Generate or use provided ID id := opts.ID if id == "" { id = petname.Generate(2, "-") @@ -54,7 +74,7 @@ func Install(cfg *config.Config, chartRef string, opts InstallOptions) error { fmt.Printf("Using deployment ID: %s\n", id) } - // 4. Check if deployment exists + // 5. Check if deployment exists deploymentDir := filepath.Join(cfg.ConfigDir, "applications", appName, id) if _, err := os.Stat(deploymentDir); err == nil { if !opts.Force { @@ -62,176 +82,150 @@ func Install(cfg *config.Config, chartRef string, opts InstallOptions) error { "Directory: %s\n"+ "Use --force or -f to overwrite", appName, id, deploymentDir) } - fmt.Printf("⚠️ WARNING: Overwriting existing deployment at %s\n", deploymentDir) + fmt.Printf("WARNING: Overwriting existing deployment at %s\n", deploymentDir) } - // 5. Create deployment directory + // 6. Create deployment directory if err := os.MkdirAll(deploymentDir, 0755); err != nil { return fmt.Errorf("failed to create deployment directory: %w", err) } - // 6. Pull chart to local directory - version := opts.Version - fetchedVersion, err := pullChart(cfg, chart, deploymentDir, version) + // 7. Fetch default values using helm show values + fmt.Printf("Fetching chart default values...\n") + values, err := fetchChartValues(cfg, chart) if err != nil { - return fmt.Errorf("failed to pull chart: %w", err) - } - - if version == "" { - version = fetchedVersion - } - fmt.Printf("Chart version: %s\n", version) - - // 7. Get default values from the pulled chart - values, err := getChartDefaultValues(deploymentDir) - if err != nil { - return fmt.Errorf("failed to get chart values: %w", err) + // Clean up on failure + os.RemoveAll(deploymentDir) + return fmt.Errorf("failed to fetch chart values: %w", err) } // 8. Write values.yaml valuesPath := filepath.Join(deploymentDir, "values.yaml") if err := os.WriteFile(valuesPath, values, 0644); err != nil { + os.RemoveAll(deploymentDir) return fmt.Errorf("failed to write values.yaml: %w", err) } - // 9. Generate helmfile.yaml (references local chart) - if err := generateHelmfile(deploymentDir, chart, appName, id, version); err != nil { + // 9. Generate helmfile.yaml (references chart remotely) + if err := generateRemoteHelmfile(deploymentDir, chart, appName, id); err != nil { + os.RemoveAll(deploymentDir) return fmt.Errorf("failed to generate helmfile: %w", err) } - // 10. Save metadata - meta := &Metadata{ - ChartURL: chart.ChartURL, - ChartName: chart.ChartName, - Version: version, - InstalledAt: time.Now(), - UpdatedAt: time.Now(), - } - if err := SaveMetadata(deploymentDir, meta); err != nil { - return fmt.Errorf("failed to save metadata: %w", err) - } - - // 11. Print success message + // 10. Print success message fmt.Printf("\n✓ Application installed successfully!\n") fmt.Printf("Deployment: %s/%s\n", appName, id) fmt.Printf("Location: %s\n", deploymentDir) - fmt.Printf("\nChart files downloaded locally:\n") - fmt.Printf(" - Chart.yaml: Chart metadata\n") + fmt.Printf("\nFiles created:\n") + fmt.Printf(" - helmfile.yaml: Deployment configuration\n") fmt.Printf(" - values.yaml: Chart default values (edit to customize)\n") - fmt.Printf(" - templates/: Kubernetes resource templates\n") - fmt.Printf(" - helmfile.yaml: Deployment definition\n") - fmt.Printf(" - metadata.yaml: Installation metadata\n") - fmt.Printf("\nYou can modify the chart templates and values directly.\n") + fmt.Printf("\nEdit values.yaml to customize your deployment.\n") fmt.Printf("To deploy, run: obol app sync %s/%s\n", appName, id) return nil } -// pullChart downloads the chart files to the deployment directory -func pullChart(cfg *config.Config, chart *ChartReference, destDir, version string) (string, error) { +// fetchChartValues retrieves default values from a chart using helm show values +func fetchChartValues(cfg *config.Config, chart *ChartReference) ([]byte, error) { helmPath := filepath.Join(cfg.BinDir, "helm") - fmt.Printf("Pulling chart: %s\n", chart.ChartURL) - - // Pull chart with helm pull --untar - args := []string{"pull", chart.ChartURL, "--untar", "--untardir", destDir} - if version != "" { - args = append(args, "--version", version) + var args []string + switch chart.Format { + case FormatURL: + // Direct URL reference + args = []string{"show", "values", chart.ChartURL} + case FormatRepoChart: + // Use repo URL directly without helm repo add + args = []string{"show", "values", chart.ChartName, "--repo", chart.RepoURL} + if chart.Version != "" { + args = append(args, "--version", chart.Version) + } + case FormatOCI: + // OCI reference + args = []string{"show", "values", chart.ChartURL} + if chart.Version != "" { + args = append(args, "--version", chart.Version) + } } cmd := exec.Command(helmPath, args...) - var stderr bytes.Buffer - cmd.Stdout = os.Stdout + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout cmd.Stderr = &stderr if err := cmd.Run(); err != nil { - return "", fmt.Errorf("helm pull failed: %w\n%s", err, stderr.String()) - } - - // Find the extracted chart directory (helm pull creates a subdirectory with chart name) - entries, err := os.ReadDir(destDir) - if err != nil { - return "", fmt.Errorf("failed to read dest directory: %w", err) - } - - var chartDir string - for _, entry := range entries { - if entry.IsDir() { - chartDir = filepath.Join(destDir, entry.Name()) - break - } - } - - if chartDir == "" { - return "", fmt.Errorf("no chart directory found after pull") - } - - // Move chart contents up one level (flatten directory structure) - // Note: Dependencies will be resolved by helmfile during sync - chartEntries, err := os.ReadDir(chartDir) - if err != nil { - return "", fmt.Errorf("failed to read chart directory: %w", err) + return nil, fmt.Errorf("helm show values failed: %w\n%s", err, stderr.String()) } - for _, entry := range chartEntries { - src := filepath.Join(chartDir, entry.Name()) - dst := filepath.Join(destDir, entry.Name()) - if err := os.Rename(src, dst); err != nil { - return "", fmt.Errorf("failed to move %s: %w", entry.Name(), err) - } + // Return empty YAML if chart has no default values + if stdout.Len() == 0 { + return []byte("# No default values in chart\n"), nil } - // Remove the now-empty chart directory - os.Remove(chartDir) + return stdout.Bytes(), nil +} - // Read version from Chart.yaml - chartYamlPath := filepath.Join(destDir, "Chart.yaml") - chartYamlData, err := os.ReadFile(chartYamlPath) - if err != nil { - return version, nil // Return requested version if can't read Chart.yaml - } +// generateRemoteHelmfile creates a helmfile.yaml that references the chart remotely +func generateRemoteHelmfile(dir string, chart *ChartReference, appName, id string) error { + var tmpl string - var chartInfo struct { - Version string `yaml:"version"` - } - if err := yaml.Unmarshal(chartYamlData, &chartInfo); err == nil { - return chartInfo.Version, nil - } + switch chart.Format { + case FormatURL: + // Direct URL reference + tmpl = `# Installed from: {{ .Original }} - return version, nil -} +releases: + - name: {{ .AppName }} + namespace: {{ .Namespace }} + createNamespace: true + chart: {{ .ChartURL }} +{{- if .Version }} + version: "{{ .Version }}" +{{- end }} + values: + - values.yaml +` + case FormatRepoChart: + // Repository reference (helmfile handles repo inline) + tmpl = `# Installed from: {{ .Original }} (resolved via ArtifactHub) -// getChartDefaultValues reads the default values from the local chart -func getChartDefaultValues(chartDir string) ([]byte, error) { - valuesPath := filepath.Join(chartDir, "values.yaml") - data, err := os.ReadFile(valuesPath) - if err != nil { - // Chart might not have a values.yaml, that's ok - if os.IsNotExist(err) { - return []byte{}, nil - } - return nil, fmt.Errorf("failed to read values.yaml: %w", err) - } - return data, nil -} +repositories: + - name: {{ .RepoName }} + url: {{ .RepoURL }} -// generateHelmfile creates the helmfile.yaml for the app -func generateHelmfile(dir string, chart *ChartReference, appName, id, version string) error { - tmpl := `# Generated by obol app install -# Chart: {{ .ChartURL }} -# Chart files are stored locally in this directory +releases: + - name: {{ .AppName }} + namespace: {{ .Namespace }} + createNamespace: true + chart: {{ .RepoName }}/{{ .ChartName }} + version: "{{ .Version }}" + values: + - values.yaml +` + case FormatOCI: + // OCI registry reference + tmpl = `# Installed from: {{ .Original }} releases: - name: {{ .AppName }} namespace: {{ .Namespace }} createNamespace: true - chart: . + chart: {{ .ChartURL }} +{{- if .Version }} + version: "{{ .Version }}" +{{- end }} values: - values.yaml ` + } data := map[string]interface{}{ + "Original": chart.Original, "ChartURL": chart.ChartURL, + "RepoName": chart.RepoName, + "RepoURL": chart.RepoURL, + "ChartName": chart.ChartName, + "Version": chart.Version, "AppName": appName, "Namespace": fmt.Sprintf("%s-%s", appName, id), } @@ -306,12 +300,6 @@ func Sync(cfg *config.Config, deploymentIdentifier string) error { return fmt.Errorf("helmfile sync failed: %w", err) } - // Update metadata timestamp - if meta, err := LoadMetadata(deploymentDir); err == nil { - meta.UpdatedAt = time.Now() - SaveMetadata(deploymentDir, meta) - } - namespace := fmt.Sprintf("%s-%s", appName, id) fmt.Printf("\n✓ Application synced successfully!\n") fmt.Printf("Namespace: %s\n", namespace) @@ -342,9 +330,9 @@ func List(cfg *config.Config, opts ListOptions) error { // Check if applications directory exists if _, err := os.Stat(appsDir); os.IsNotExist(err) { fmt.Println("No applications installed") - fmt.Println("\nTo install an application, provide a chart URL:") + fmt.Println("\nTo install an application:") + fmt.Println(" obol app install bitnami/redis") fmt.Println(" obol app install https://charts.bitnami.com/bitnami/redis-19.0.0.tgz") - fmt.Println(" obol app install https://charts.bitnami.com/bitnami/postgresql-15.0.0.tgz") fmt.Println("\nFind charts at https://artifacthub.io") return nil } @@ -386,10 +374,10 @@ func List(cfg *config.Config, opts ListOptions) error { id := deployment.Name() deploymentPath := filepath.Join(appPath, id) - // Load metadata - meta, err := LoadMetadata(deploymentPath) + // Parse helmfile for chart info + info, err := ParseHelmfile(deploymentPath) if err != nil { - // Metadata not found - show basic info + // Helmfile not found - show basic info fmt.Printf(" %s/%s\n", appName, id) count++ continue @@ -398,16 +386,15 @@ func List(cfg *config.Config, opts ListOptions) error { // Show deployment info if opts.Verbose { fmt.Printf(" %s/%s\n", appName, id) - fmt.Printf(" Chart: %s\n", meta.ChartURL) - fmt.Printf(" Version: %s\n", meta.Version) - fmt.Printf(" Installed: %s\n", meta.InstalledAt.Format("2006-01-02 15:04:05")) - if !meta.UpdatedAt.IsZero() && meta.UpdatedAt != meta.InstalledAt { - fmt.Printf(" Updated: %s\n", meta.UpdatedAt.Format("2006-01-02 15:04:05")) + fmt.Printf(" Chart: %s\n", info.ChartRef) + fmt.Printf(" Version: %s\n", info.Version) + if modTime, err := GetHelmfileModTime(deploymentPath); err == nil { + fmt.Printf(" Modified: %s\n", modTime) } fmt.Println() } else { fmt.Printf(" %s/%s (chart: %s, version: %s)\n", - appName, id, meta.ChartURL, meta.Version) + appName, id, info.ChartRef, info.Version) } count++ } diff --git a/internal/app/artifacthub.go b/internal/app/artifacthub.go new file mode 100644 index 0000000..cacc31c --- /dev/null +++ b/internal/app/artifacthub.go @@ -0,0 +1,134 @@ +package app + +import ( + "encoding/json" + "fmt" + "net/http" + "strings" + "time" +) + +// ArtifactHubClient handles chart resolution via the ArtifactHub API +type ArtifactHubClient struct { + httpClient *http.Client + baseURL string +} + +// ArtifactHubPackage represents the API response for a package +type ArtifactHubPackage struct { + Name string `json:"name"` + Version string `json:"version"` + Repository ArtifactHubRepository `json:"repository"` + ContentURL string `json:"content_url"` + AvailableVersions []ArtifactHubVersion `json:"available_versions"` +} + +// ArtifactHubRepository represents repository info in the API response +type ArtifactHubRepository struct { + Name string `json:"name"` + URL string `json:"url"` +} + +// ArtifactHubVersion represents a version in the available_versions list +type ArtifactHubVersion struct { + Version string `json:"version"` +} + +// ChartInfo holds resolved chart information from ArtifactHub +type ChartInfo struct { + RepoName string // Repository name (e.g., "bitnami") + RepoURL string // Repository URL (e.g., "https://charts.bitnami.com/bitnami") + ChartName string // Chart name (e.g., "redis") + Version string // Resolved version (e.g., "19.6.4") +} + +// NewArtifactHubClient creates a new ArtifactHub API client +func NewArtifactHubClient() *ArtifactHubClient { + return &ArtifactHubClient{ + httpClient: &http.Client{ + Timeout: 30 * time.Second, + }, + baseURL: "https://artifacthub.io/api/v1", + } +} + +// ResolveChart resolves a repo/chart[@version] reference to full chart info +// Input formats: +// - "bitnami/redis" -> resolves to latest version +// - "bitnami/redis@19.0.0" -> resolves to specific version +func (c *ArtifactHubClient) ResolveChart(ref string) (*ChartInfo, error) { + // Parse the reference + repoName, chartName, version, err := parseRepoChartRef(ref) + if err != nil { + return nil, err + } + + // Build API URL + var apiURL string + if version != "" { + // Specific version endpoint + apiURL = fmt.Sprintf("%s/packages/helm/%s/%s/%s", c.baseURL, repoName, chartName, version) + } else { + // Latest version endpoint + apiURL = fmt.Sprintf("%s/packages/helm/%s/%s", c.baseURL, repoName, chartName) + } + + // Make request + resp, err := c.httpClient.Get(apiURL) + if err != nil { + return nil, fmt.Errorf("failed to query ArtifactHub: %w\n"+ + "Please check your network connection or provide a direct chart URL instead", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + if version != "" { + return nil, fmt.Errorf("chart not found: %s/%s version %s\n"+ + "Verify the repository, chart name, and version are correct on https://artifacthub.io", repoName, chartName, version) + } + return nil, fmt.Errorf("chart not found: %s/%s\n"+ + "Verify the repository and chart name are correct on https://artifacthub.io", repoName, chartName) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("ArtifactHub API error (status %d)\n"+ + "Please try again later or provide a direct chart URL instead", resp.StatusCode) + } + + // Parse response + var pkg ArtifactHubPackage + if err := json.NewDecoder(resp.Body).Decode(&pkg); err != nil { + return nil, fmt.Errorf("failed to parse ArtifactHub response: %w", err) + } + + // Validate we got the required fields + if pkg.Repository.URL == "" { + return nil, fmt.Errorf("ArtifactHub returned incomplete data: missing repository URL") + } + + return &ChartInfo{ + RepoName: repoName, + RepoURL: pkg.Repository.URL, + ChartName: chartName, + Version: pkg.Version, + }, nil +} + +// parseRepoChartRef parses "repo/chart[@version]" format +func parseRepoChartRef(ref string) (repoName, chartName, version string, err error) { + // Check for version suffix + if idx := strings.LastIndex(ref, "@"); idx != -1 { + version = ref[idx+1:] + ref = ref[:idx] + } + + // Split repo/chart + parts := strings.SplitN(ref, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", "", fmt.Errorf("invalid chart reference: %s\n"+ + "Expected format: repo/chart or repo/chart@version\n"+ + "Example: bitnami/redis or bitnami/redis@19.0.0", ref) + } + + return parts[0], parts[1], version, nil +} diff --git a/internal/app/chart.go b/internal/app/chart.go index 054e81f..678a16a 100644 --- a/internal/app/chart.go +++ b/internal/app/chart.go @@ -8,26 +8,56 @@ import ( "strings" ) +// ChartFormat represents the type of chart reference +type ChartFormat int + +const ( + FormatURL ChartFormat = iota // https://.../*.tgz + FormatRepoChart // repo/chart[@version] + FormatOCI // oci://... +) + // ChartReference holds parsed chart information type ChartReference struct { - Original string // Original input string - ChartName string // Chart name extracted from URL - ChartURL string // Full URL to chart + Original string // Original input string + Format ChartFormat // Detected format type + ChartName string // Chart name + ChartURL string // Full URL (for URL/OCI formats) + RepoName string // Repository name (for repo/chart format) + RepoURL string // Repository URL (for repo/chart format, resolved) + Version string // Chart version (may be empty) } -// ParseChartReference parses a chart URL +// ParseChartReference parses a chart reference in any supported format func ParseChartReference(ref string) (*ChartReference, error) { ref = strings.TrimSpace(ref) - // Only support HTTPS/HTTP URLs - if !strings.HasPrefix(ref, "https://") && !strings.HasPrefix(ref, "http://") { - return nil, fmt.Errorf("invalid chart URL: %s\n"+ - "Please provide a direct HTTPS URL to a chart .tgz file\n"+ - "Example: https://charts.bitnami.com/bitnami/redis-19.0.0.tgz\n"+ - "Find chart URLs at https://artifacthub.io", ref) + // Detect format + switch { + case strings.HasPrefix(ref, "oci://"): + return parseOCIReference(ref) + case strings.HasPrefix(ref, "https://") || strings.HasPrefix(ref, "http://"): + return parseURLReference(ref) + case isRepoChartFormat(ref): + return parseRepoChartReference(ref) + default: + return nil, fmt.Errorf("invalid chart reference: %s\n\n"+ + "Supported formats:\n"+ + " URL: https://charts.bitnami.com/bitnami/redis-19.0.0.tgz\n"+ + " Repo/Chart: bitnami/redis or bitnami/redis@19.0.0\n"+ + " OCI: oci://registry-1.docker.io/bitnamicharts/redis\n\n"+ + "Find charts at https://artifacthub.io", ref) } +} - return parseURLReference(ref) +// isRepoChartFormat checks if the reference looks like repo/chart[@version] +func isRepoChartFormat(ref string) bool { + // Must contain exactly one slash (not counting any in version part) + base := ref + if idx := strings.LastIndex(ref, "@"); idx != -1 { + base = ref[:idx] + } + return strings.Count(base, "/") == 1 && !strings.Contains(ref, "://") } func parseURLReference(ref string) (*ChartReference, error) { @@ -36,22 +66,80 @@ func parseURLReference(ref string) (*ChartReference, error) { return nil, fmt.Errorf("invalid URL: %w", err) } - // Extract chart name from URL path (e.g., redis-19.0.0.tgz -> redis) + // Extract chart name and version from URL path path := u.Path base := filepath.Base(path) // Remove .tgz extension - chartName := strings.TrimSuffix(base, ".tgz") - // Remove version suffix (e.g., redis-19.0.0 -> redis) - re := regexp.MustCompile(`^(.+)-\d+\.\d+\.\d+.*$`) - if matches := re.FindStringSubmatch(chartName); len(matches) > 1 { + nameWithVersion := strings.TrimSuffix(base, ".tgz") + + // Extract chart name and version (e.g., redis-19.0.0 -> redis, 19.0.0) + chartName := nameWithVersion + version := "" + re := regexp.MustCompile(`^(.+)-(\d+\.\d+\.\d+.*)$`) + if matches := re.FindStringSubmatch(nameWithVersion); len(matches) > 2 { chartName = matches[1] + version = matches[2] } return &ChartReference{ Original: ref, + Format: FormatURL, ChartName: chartName, ChartURL: ref, + Version: version, + }, nil +} + +func parseOCIReference(ref string) (*ChartReference, error) { + // oci://registry-1.docker.io/bitnamicharts/redis + // oci://registry-1.docker.io/bitnamicharts/redis:19.0.0 + + withoutScheme := strings.TrimPrefix(ref, "oci://") + parts := strings.Split(withoutScheme, "/") + if len(parts) < 2 { + return nil, fmt.Errorf("invalid OCI reference: %s", ref) + } + + // Last part is chart name (possibly with :version) + last := parts[len(parts)-1] + chartName := last + version := "" + if idx := strings.LastIndex(last, ":"); idx != -1 { + chartName = last[:idx] + version = last[idx+1:] + } + + return &ChartReference{ + Original: ref, + Format: FormatOCI, + ChartName: chartName, + ChartURL: ref, + Version: version, + }, nil +} + +func parseRepoChartReference(ref string) (*ChartReference, error) { + // Parse repo/chart[@version] + version := "" + base := ref + if idx := strings.LastIndex(ref, "@"); idx != -1 { + version = ref[idx+1:] + base = ref[:idx] + } + + parts := strings.SplitN(base, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return nil, fmt.Errorf("invalid repo/chart reference: %s", ref) + } + + return &ChartReference{ + Original: ref, + Format: FormatRepoChart, + ChartName: parts[1], + RepoName: parts[0], + Version: version, + // RepoURL will be resolved via ArtifactHub }, nil } @@ -59,3 +147,8 @@ func parseURLReference(ref string) (*ChartReference, error) { func (c *ChartReference) GetChartName() string { return c.ChartName } + +// NeedsResolution returns true if this reference needs ArtifactHub resolution +func (c *ChartReference) NeedsResolution() bool { + return c.Format == FormatRepoChart +} diff --git a/internal/app/metadata.go b/internal/app/metadata.go index 63ad435..c4bf035 100644 --- a/internal/app/metadata.go +++ b/internal/app/metadata.go @@ -1,41 +1,70 @@ package app import ( + "bufio" "os" "path/filepath" - "time" + "strings" "gopkg.in/yaml.v3" ) -// Metadata stores information about an installed application -type Metadata struct { - ChartURL string `yaml:"chartUrl"` // Chart download URL - ChartName string `yaml:"chartName"` // Extracted chart name - Version string `yaml:"version"` // Chart version - InstalledAt time.Time `yaml:"installedAt"` // Installation timestamp - UpdatedAt time.Time `yaml:"updatedAt,omitempty"` // Last update timestamp +// HelmfileInfo holds parsed information from a helmfile.yaml +type HelmfileInfo struct { + ChartRef string // Original chart reference (from comment) + Chart string // Chart field value + Version string // Chart version } -// SaveMetadata writes metadata to the deployment directory -func SaveMetadata(dir string, meta *Metadata) error { - data, err := yaml.Marshal(meta) +// ParseHelmfile extracts chart information from a helmfile.yaml +func ParseHelmfile(dir string) (*HelmfileInfo, error) { + helmfilePath := filepath.Join(dir, "helmfile.yaml") + data, err := os.ReadFile(helmfilePath) if err != nil { - return err + return nil, err } - return os.WriteFile(filepath.Join(dir, "metadata.yaml"), data, 0644) -} -// LoadMetadata reads metadata from a deployment directory -func LoadMetadata(dir string) (*Metadata, error) { - data, err := os.ReadFile(filepath.Join(dir, "metadata.yaml")) - if err != nil { - return nil, err + info := &HelmfileInfo{} + + // Extract original reference from first comment line + scanner := bufio.NewScanner(strings.NewReader(string(data))) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "# Installed from: ") { + info.ChartRef = strings.TrimPrefix(line, "# Installed from: ") + // Remove any trailing annotation like "(resolved via ArtifactHub)" + if idx := strings.Index(info.ChartRef, " ("); idx != -1 { + info.ChartRef = info.ChartRef[:idx] + } + break + } } - var meta Metadata - if err := yaml.Unmarshal(data, &meta); err != nil { - return nil, err + // Parse YAML structure to get chart and version + var helmfile struct { + Releases []struct { + Chart string `yaml:"chart"` + Version string `yaml:"version"` + } `yaml:"releases"` + } + if err := yaml.Unmarshal(data, &helmfile); err != nil { + return info, nil // Return partial info if YAML parsing fails + } + + if len(helmfile.Releases) > 0 { + info.Chart = helmfile.Releases[0].Chart + info.Version = helmfile.Releases[0].Version + } + + return info, nil +} + +// GetHelmfileModTime returns the modification time of helmfile.yaml +func GetHelmfileModTime(dir string) (modTime string, err error) { + helmfilePath := filepath.Join(dir, "helmfile.yaml") + stat, err := os.Stat(helmfilePath) + if err != nil { + return "", err } - return &meta, nil + return stat.ModTime().Format("2006-01-02 15:04:05"), nil } From 69f79582f3f06816effdb857e8032e02882c0d63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A1draic=20=C3=93=20Mhuiris?= Date: Wed, 10 Dec 2025 22:29:50 +0000 Subject: [PATCH 5/5] updated readme --- README.md | 46 +++++++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index e34b6a4..201b3ef 100644 --- a/README.md +++ b/README.md @@ -129,7 +129,7 @@ The stack will create a local Kubernetes cluster. Each network installation crea > Use `obol network list` to see all available networks. Customize installations with flags (e.g., `obol network install ethereum --network=holesky --execution-client=geth`) to create different deployment configurations. After installation, deploy to the cluster with `obol network sync /`. > [!TIP] -> You can also install arbitrary Helm charts as applications using `obol app install `. Find charts at [Artifact Hub](https://artifacthub.io). +> You can also install arbitrary Helm charts as applications using `obol app install `. Find charts at [Artifact Hub](https://artifacthub.io). ## Managing Networks @@ -276,23 +276,32 @@ The Obol Stack supports installing arbitrary Helm charts as managed applications ### Install an Application -Install any Helm chart by providing a direct HTTPS URL to a chart `.tgz` file: +Install any Helm chart using one of the supported reference formats: ```bash -# Install Redis from Bitnami +# Install using repo/chart format (resolved via ArtifactHub) +obol app install bitnami/redis +obol app install bitnami/postgresql@15.0.0 + +# Install using direct URL obol app install https://charts.bitnami.com/bitnami/redis-19.0.0.tgz -# Install PostgreSQL with custom name and ID -obol app install https://charts.bitnami.com/bitnami/postgresql-15.0.0.tgz --name mydb --id production +# Install with custom name and ID +obol app install bitnami/postgresql --name mydb --id production ``` -Find chart URLs at [Artifact Hub](https://artifacthub.io). +Find charts at [Artifact Hub](https://artifacthub.io). + +**Supported chart reference formats:** +- `repo/chart` - Resolved via ArtifactHub (e.g., `bitnami/redis`) +- `repo/chart@version` - Specific version (e.g., `bitnami/redis@19.0.0`) +- `https://.../*.tgz` - Direct URL to chart archive **What happens during installation:** -1. Downloads the chart files locally to `~/.config/obol/applications///` -2. Extracts chart templates, values, and metadata -3. Generates a helmfile.yaml for deployment -4. Saves installation metadata +1. Resolves the chart reference (via ArtifactHub for repo/chart format) +2. Fetches default values from the chart +3. Generates helmfile.yaml that references the chart remotely +4. Saves configuration to `~/.config/obol/applications///` **Installation options:** - `--name`: Application name (defaults to chart name) @@ -347,25 +356,19 @@ This command will: ### Customize Applications -After installation, you can modify the chart files directly: +After installation, you can modify the values file to customize your deployment: ```bash # Edit application values $EDITOR ~/.config/obol/applications/postgresql/eager-fox/values.yaml -# Modify templates -$EDITOR ~/.config/obol/applications/postgresql/eager-fox/templates/statefulset.yaml - # Re-deploy with changes obol app sync postgresql/eager-fox ``` **Local files:** -- `Chart.yaml`: Chart metadata +- `helmfile.yaml`: Deployment configuration (references chart remotely) - `values.yaml`: Configuration values (edit to customize) -- `templates/`: Kubernetes resource templates -- `helmfile.yaml`: Deployment definition -- `metadata.yaml`: Installation metadata ### Managing the Stack @@ -496,11 +499,8 @@ The Obol Stack follows the [XDG Base Directory](https://specifications.freedeskt └── applications/ # Installed application deployments ├── redis/ # Redis deployments │ └── / # Deployment instance - │ ├── Chart.yaml # Chart metadata - │ ├── values.yaml # Configuration values - │ ├── helmfile.yaml # Deployment definition - │ ├── metadata.yaml # Installation metadata - │ └── templates/ # Kubernetes resources + │ ├── helmfile.yaml # Deployment configuration + │ └── values.yaml # Configuration values └── postgresql/ # PostgreSQL deployments └── / # Deployment instance ```