Correlations: Add CreateCorrelation HTTP API (grafana#51630)

* Correlations: add migration

* Correlations: Add CreateCorrelation API

* Correlations: Make correlations work with provisioning

* Handle version changes

* Fix lining error

* lint fixes

* rebuild betterer results

* add a UID to each correlation

* Fix lint errors

* add docs

* better wording in API docs

* remove leftover comment

* handle ds updates

* Fix error message typo

* add bad data test

* make correlations a separate table

* skip readonly check when provisioning correlations

* delete stale correlations when datasources are deleted

* restore provisioned readonly ds

* publish deletion event with full data

* generate swagger and HTTP API docs

* apply source datasource permission to create correlation API

* Fix tests & lint errors

* ignore empty deletion events

* fix last lint errors

* fix more lint error

* Only publish deletion event if datasource was actually deleted

* delete DS provisioning deletes correlations, added & fixed tests

* Fix unmarshalling tests

* Fix linting errors

* Fix deltion event tests

* fix small linting error

* fix lint errors

* update betterer

* fix test

* make path singular

* Revert "make path singular"

This reverts commit 420c3d3.

* add integration tests

* remove unneeded id from correlations table

* update spec

* update leftover references to CorrelationDTO

* fix tests

* cleanup tests

* fix lint error

Author: Elfo404

devenv/datasources.yaml

@@ -247,7 +247,15 @@ datasources:
     access: proxy
     url: http://localhost:3100
     editable: false
+    correlations:
+      - targetUid: gdev-jaeger
+        label: "Jaeger traces"
+        description: "Related traces stored in Jaeger"
+      - targetUid: gdev-zipkin
+        label: "Zipkin traces"
+        description: "Related traces stored in Zipkin"
     jsonData:
+      something: here
       manageAlerts: false
       derivedFields:
         - name: "traceID"

docs/sources/developers/http_api/correlations.md

@@ -0,0 +1,70 @@
+---
+aliases:
+  - /docs/grafana/latest/developers/http_api/correlations/
+  - /docs/grafana/latest/http_api/correlations/
+description: Grafana Correlations HTTP API
+keywords:
+  - grafana
+  - http
+  - documentation
+  - api
+  - correlations
+  - Glue
+title: 'Correlations HTTP API '
+---
+
+# Correlations API
+
+This API can be used to define correlations between data sources.
+
+## Create correlations
+
+`POST /api/datasources/uid/:sourceUid/correlations`
+
+Creates a correlation between two data sources - the source data source indicated by the path UID, and the target data source which is specified in the body.
+
+**Example request:**
+
+```http
+POST /api/datasources/uid/uyBf2637k/correlations HTTP/1.1
+Accept: application/json
+Content-Type: application/json
+Authorization: Bearer eyJrIjoiT0tTcG1pUlY2RnVKZTFVaDFsNFZXdE9ZWmNrMkZYbk
+{
+	"targetUid": "PDDA8E780A17E7EF1",
+	"label": "My Label",
+	"description": "Logs to Traces",
+}
+```
+
+JSON body schema:
+
+- **targetUid** – Target data source uid.
+- **label** – A label for the correlation.
+- **description** – A description for the correlation.
+
+**Example response:**
+
+```http
+HTTP/1.1 200
+Content-Type: application/json
+{
+  "message": "Correlation created",
+  "result": {
+    "description": "Logs to Traces",
+    "label": "My Label",
+    "sourceUid": "uyBf2637k",
+    "targetUid": "PDDA8E780A17E7EF1",
+    "uid": "50xhMlg9k"
+  }
+}
+```
+
+Status codes:
+
+- **200** – OK
+- **400** - Errors (invalid JSON, missing or invalid fields)
+- **401** – Unauthorized
+- **403** – Forbidden, source data source is read-only
+- **404** – Not found, either source or target data source could not be found
+- **500** – Internal error

pkg/api/docs/definitions/correlations.go

@@ -0,0 +1,33 @@
+package definitions
+
+import (
+	"github.com/grafana/grafana/pkg/services/correlations"
+)
+
+// swagger:route POST /datasources/uid/{uid}/correlations correlations createCorrelation
+//
+// Add correlation.
+//
+// Responses:
+// 200: createCorrelationResponse
+// 400: badRequestError
+// 401: unauthorisedError
+// 403: forbiddenError
+// 404: notFoundError
+// 500: internalServerError
+
+// swagger:parameters createCorrelation
+type CreateCorrelationParams struct {
+	// in:body
+	// required:true
+	Body correlations.CreateCorrelationCommand `json:"body"`
+	// in:path
+	// required:true
+	SourceUID string `json:"uid"`
+}
+
+//swagger:response createCorrelationResponse
+type CreateCorrelationResponse struct {
+	// in: body
+	Body correlations.CreateCorrelationResponse `json:"body"`
+}

pkg/api/http_server.go

@@ -65,6 +65,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/provisioning"
 	"github.com/grafana/grafana/pkg/services/quota"
 
+	"github.com/grafana/grafana/pkg/services/correlations"
 	publicdashboardsApi "github.com/grafana/grafana/pkg/services/publicdashboards/api"
 	"github.com/grafana/grafana/pkg/services/query"
 	"github.com/grafana/grafana/pkg/services/queryhistory"
@@ -122,6 +123,7 @@ type HTTPServer struct {
 	SearchService                search.Service
 	ShortURLService              shorturls.Service
 	QueryHistoryService          queryhistory.Service
+	CorrelationsService          correlations.Service
 	Live                         *live.GrafanaLive
 	LivePushGateway              *pushhttp.Gateway
 	ThumbService                 thumbs.Service
@@ -188,7 +190,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	pluginDashboardService plugindashboards.Service, pluginStore plugins.Store, pluginClient plugins.Client,
 	pluginErrorResolver plugins.ErrorResolver, pluginManager plugins.Manager, settingsProvider setting.Provider,
 	dataSourceCache datasources.CacheService, userTokenService models.UserTokenService,
-	cleanUpService *cleanup.CleanUpService, shortURLService shorturls.Service, queryHistoryService queryhistory.Service,
+	cleanUpService *cleanup.CleanUpService, shortURLService shorturls.Service, queryHistoryService queryhistory.Service, correlationsService correlations.Service,
 	thumbService thumbs.Service, remoteCache *remotecache.RemoteCache, provisioningService provisioning.ProvisioningService,
 	loginService login.Service, authenticator loginpkg.Authenticator, accessControl accesscontrol.AccessControl,
 	dataSourceProxy *datasourceproxy.DataSourceProxyService, searchService *search.SearchService,
@@ -239,6 +241,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		cleanUpService:               cleanUpService,
 		ShortURLService:              shortURLService,
 		QueryHistoryService:          queryHistoryService,
+		CorrelationsService:          correlationsService,
 		Features:                     features,
 		ThumbService:                 thumbService,
 		StorageService:               storageService,

pkg/server/wire.go

@@ -45,6 +45,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/comments"
 	"github.com/grafana/grafana/pkg/services/contexthandler"
 	"github.com/grafana/grafana/pkg/services/contexthandler/authproxy"
+	"github.com/grafana/grafana/pkg/services/correlations"
 	"github.com/grafana/grafana/pkg/services/dashboardimport"
 	dashboardimportservice "github.com/grafana/grafana/pkg/services/dashboardimport/service"
 	"github.com/grafana/grafana/pkg/services/dashboards"
@@ -183,6 +184,8 @@ var wireBasicSet = wire.NewSet(
 	wire.Bind(new(shorturls.Service), new(*shorturls.ShortURLService)),
 	queryhistory.ProvideService,
 	wire.Bind(new(queryhistory.Service), new(*queryhistory.QueryHistoryService)),
+	correlations.ProvideService,
+	wire.Bind(new(correlations.Service), new(*correlations.CorrelationsService)),
 	quotaimpl.ProvideService,
 	remotecache.ProvideService,
 	loginservice.ProvideService,

pkg/services/correlations/api.go

@@ -0,0 +1,49 @@
+package correlations
+
+import (
+	"errors"
+	"net/http"
+
+	"github.com/grafana/grafana/pkg/api/response"
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/middleware"
+	"github.com/grafana/grafana/pkg/models"
+	ac "github.com/grafana/grafana/pkg/services/accesscontrol"
+	"github.com/grafana/grafana/pkg/services/datasources"
+
+	"github.com/grafana/grafana/pkg/web"
+)
+
+func (s *CorrelationsService) registerAPIEndpoints() {
+	uidScope := datasources.ScopeProvider.GetResourceScopeUID(ac.Parameter(":uid"))
+	authorize := ac.Middleware(s.AccessControl)
+
+	s.RouteRegister.Group("/api/datasources/uid/:uid/correlations", func(entities routing.RouteRegister) {
+		entities.Post("/", middleware.ReqSignedIn, authorize(ac.ReqOrgAdmin, ac.EvalPermission(datasources.ActionWrite, uidScope)), routing.Wrap(s.createHandler))
+	})
+}
+
+// createHandler handles POST /datasources/uid/:uid/correlations
+func (s *CorrelationsService) createHandler(c *models.ReqContext) response.Response {
+	cmd := CreateCorrelationCommand{}
+	if err := web.Bind(c.Req, &cmd); err != nil {
+		return response.Error(http.StatusBadRequest, "bad request data", err)
+	}
+	cmd.SourceUID = web.Params(c.Req)[":uid"]
+	cmd.OrgId = c.OrgId
+
+	correlation, err := s.CreateCorrelation(c.Req.Context(), cmd)
+	if err != nil {
+		if errors.Is(err, ErrSourceDataSourceDoesNotExists) || errors.Is(err, ErrTargetDataSourceDoesNotExists) {
+			return response.Error(http.StatusNotFound, "Data source not found", err)
+		}
+
+		if errors.Is(err, ErrSourceDataSourceReadOnly) {
+			return response.Error(http.StatusForbidden, "Data source is read only", err)
+		}
+
+		return response.Error(http.StatusInternalServerError, "Failed to add correlation", err)
+	}
+
+	return response.JSON(http.StatusOK, CreateCorrelationResponse{Result: correlation, Message: "Correlation created"})
+}

pkg/services/correlations/correlations.go

@@ -0,0 +1,74 @@
+package correlations
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/events"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/accesscontrol"
+	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+)
+
+func ProvideService(sqlStore *sqlstore.SQLStore, routeRegister routing.RouteRegister, ds datasources.DataSourceService, ac accesscontrol.AccessControl, bus bus.Bus) *CorrelationsService {
+	s := &CorrelationsService{
+		SQLStore:          sqlStore,
+		RouteRegister:     routeRegister,
+		log:               log.New("correlations"),
+		DataSourceService: ds,
+		AccessControl:     ac,
+	}
+
+	s.registerAPIEndpoints()
+
+	bus.AddEventListener(s.handleDatasourceDeletion)
+
+	return s
+}
+
+type Service interface {
+	CreateCorrelation(ctx context.Context, cmd CreateCorrelationCommand) (Correlation, error)
+	DeleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error
+	DeleteCorrelationsByTargetUID(ctx context.Context, cmd DeleteCorrelationsByTargetUIDCommand) error
+}
+
+type CorrelationsService struct {
+	SQLStore          *sqlstore.SQLStore
+	RouteRegister     routing.RouteRegister
+	log               log.Logger
+	DataSourceService datasources.DataSourceService
+	AccessControl     accesscontrol.AccessControl
+}
+
+func (s CorrelationsService) CreateCorrelation(ctx context.Context, cmd CreateCorrelationCommand) (Correlation, error) {
+	return s.createCorrelation(ctx, cmd)
+}
+
+func (s CorrelationsService) DeleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error {
+	return s.deleteCorrelationsBySourceUID(ctx, cmd)
+}
+
+func (s CorrelationsService) DeleteCorrelationsByTargetUID(ctx context.Context, cmd DeleteCorrelationsByTargetUIDCommand) error {
+	return s.deleteCorrelationsByTargetUID(ctx, cmd)
+}
+
+func (s CorrelationsService) handleDatasourceDeletion(ctx context.Context, event *events.DataSourceDeleted) error {
+	return s.SQLStore.InTransaction(ctx, func(ctx context.Context) error {
+		if err := s.deleteCorrelationsBySourceUID(ctx, DeleteCorrelationsBySourceUIDCommand{
+			SourceUID: event.UID,
+		}); err != nil {
+			return err
+		}
+
+		if err := s.deleteCorrelationsByTargetUID(ctx, DeleteCorrelationsByTargetUIDCommand{
+			TargetUID: event.UID,
+		}); err != nil {
+			return err
+		}
+
+		return nil
+	})
+}

pkg/services/correlations/database.go

@@ -0,0 +1,70 @@
+package correlations
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"github.com/grafana/grafana/pkg/util"
+)
+
+// createCorrelation adds a correlation
+func (s CorrelationsService) createCorrelation(ctx context.Context, cmd CreateCorrelationCommand) (Correlation, error) {
+	correlation := Correlation{
+		UID:         util.GenerateShortUID(),
+		SourceUID:   cmd.SourceUID,
+		TargetUID:   cmd.TargetUID,
+		Label:       cmd.Label,
+		Description: cmd.Description,
+	}
+
+	err := s.SQLStore.WithTransactionalDbSession(ctx, func(session *sqlstore.DBSession) error {
+		var err error
+
+		query := &datasources.GetDataSourceQuery{
+			OrgId: cmd.OrgId,
+			Uid:   cmd.SourceUID,
+		}
+		if err = s.DataSourceService.GetDataSource(ctx, query); err != nil {
+			return ErrSourceDataSourceDoesNotExists
+		}
+
+		if !cmd.SkipReadOnlyCheck && query.Result.ReadOnly {
+			return ErrSourceDataSourceReadOnly
+		}
+
+		if err = s.DataSourceService.GetDataSource(ctx, &datasources.GetDataSourceQuery{
+			OrgId: cmd.OrgId,
+			Uid:   cmd.TargetUID,
+		}); err != nil {
+			return ErrTargetDataSourceDoesNotExists
+		}
+
+		_, err = session.Insert(correlation)
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return Correlation{}, err
+	}
+
+	return correlation, nil
+}
+
+func (s CorrelationsService) deleteCorrelationsBySourceUID(ctx context.Context, cmd DeleteCorrelationsBySourceUIDCommand) error {
+	return s.SQLStore.WithDbSession(ctx, func(session *sqlstore.DBSession) error {
+		_, err := session.Delete(&Correlation{SourceUID: cmd.SourceUID})
+		return err
+	})
+}
+
+func (s CorrelationsService) deleteCorrelationsByTargetUID(ctx context.Context, cmd DeleteCorrelationsByTargetUIDCommand) error {
+	return s.SQLStore.WithDbSession(ctx, func(session *sqlstore.DBSession) error {
+		_, err := session.Delete(&Correlation{TargetUID: cmd.TargetUID})
+		return err
+	})
+}