Files
ocis/services/graph/pkg/service/v0/driveitems.go
Dominik Schmidt db32fb46f9 Audio Metadata (#7490)
* Add audio facet to search protobuf message

* Add audio metadata to search index

* Return audio facet from search if available

* Store audio metadata in arbitrary metadata

* Add audio facet to driveItems listings

* Make tests coding style more consistent

* Fix tests

* Add changelog

* Make valueToString code more defensive

* Log status code as well
2023-11-06 08:56:46 +01:00

573 lines
19 KiB
Go

package svc
import (
"context"
"errors"
"fmt"
"net/http"
"net/url"
"path"
"reflect"
"strconv"
"strings"
"time"
cs3rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
storageprovider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
types "github.com/cs3org/go-cs3apis/cs3/types/v1beta1"
revactx "github.com/cs3org/reva/v2/pkg/ctx"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/cs3org/reva/v2/pkg/utils"
"github.com/go-chi/render"
libregraph "github.com/owncloud/libre-graph-api-go"
"github.com/owncloud/ocis/v2/ocis-pkg/log"
"github.com/owncloud/ocis/v2/services/graph/pkg/service/v0/errorcode"
"golang.org/x/crypto/sha3"
)
// GetRootDriveChildren implements the Service interface.
func (g Graph) GetRootDriveChildren(w http.ResponseWriter, r *http.Request) {
g.logger.Info().Msg("Calling GetRootDriveChildren")
ctx := r.Context()
gatewayClient, err := g.gatewaySelector.Next()
if err != nil {
g.logger.Error().Err(err).Msg("could not select next gateway client")
errorcode.ServiceNotAvailable.Render(w, r, http.StatusInternalServerError, "could not select next gateway client, aborting")
return
}
currentUser := revactx.ContextMustGetUser(r.Context())
// do we need to list all or only the personal drive
filters := []*storageprovider.ListStorageSpacesRequest_Filter{}
filters = append(filters, listStorageSpacesUserFilter(currentUser.GetId().OpaqueId))
filters = append(filters, listStorageSpacesTypeFilter("personal"))
res, err := gatewayClient.ListStorageSpaces(ctx, &storageprovider.ListStorageSpacesRequest{
Filters: filters,
})
switch {
case err != nil:
g.logger.Error().Err(err).Msg("error making ListStorageSpaces grpc call")
errorcode.ServiceNotAvailable.Render(w, r, http.StatusInternalServerError, err.Error())
return
case res.Status.Code != cs3rpc.Code_CODE_OK:
if res.Status.Code == cs3rpc.Code_CODE_NOT_FOUND {
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, res.Status.Message)
return
}
g.logger.Error().Err(err).Msg("error sending ListStorageSpaces grpc request")
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, res.Status.Message)
return
}
var space *storageprovider.StorageSpace
for _, s := range res.StorageSpaces {
if utils.UserIDEqual(currentUser.GetId(), s.GetOwner().GetId()) {
space = s
}
}
lRes, err := gatewayClient.ListContainer(ctx, &storageprovider.ListContainerRequest{
Ref: &storageprovider.Reference{ResourceId: space.Root},
})
switch {
case err != nil:
g.logger.Error().Err(err).Msg("error making ListContainer grpc call")
errorcode.ServiceNotAvailable.Render(w, r, http.StatusInternalServerError, err.Error())
return
case lRes.Status.Code != cs3rpc.Code_CODE_OK:
if lRes.Status.Code == cs3rpc.Code_CODE_NOT_FOUND {
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, lRes.Status.Message)
return
}
if lRes.Status.Code == cs3rpc.Code_CODE_PERMISSION_DENIED {
// TODO check if we should return 404 to not disclose existing items
errorcode.AccessDenied.Render(w, r, http.StatusForbidden, lRes.Status.Message)
return
}
g.logger.Error().Err(err).Msg("error sending list container grpc request")
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, res.Status.Message)
return
}
files, err := formatDriveItems(g.logger, lRes.Infos)
if err != nil {
g.logger.Error().Err(err).Msg("error encoding response as json")
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
}
render.Status(r, http.StatusOK)
render.JSON(w, r, &ListResponse{Value: files})
}
// GetDriveItem returns a driveItem
func (g Graph) GetDriveItem(w http.ResponseWriter, r *http.Request) {
g.logger.Info().Msg("Calling GetDriveItem")
ctx := r.Context()
driveID, err := parseIDParam(r, "driveID")
if err != nil {
errorcode.RenderError(w, r, err)
return
}
driveItemID, err := parseIDParam(r, "driveItemID")
if err != nil {
errorcode.RenderError(w, r, err)
return
}
if driveID.StorageId != driveItemID.StorageId || driveID.SpaceId != driveItemID.SpaceId {
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, "Item does not exist")
return
}
/*
sanitizedPath := strings.TrimPrefix(r.URL.Path, "/graph/v1.0/")
// Parse the request with odata parser
odataReq, err := godata.ParseRequest(ctx, sanitizedPath, r.URL.Query())
if err != nil {
errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, err.Error())
return
}
*/
gatewayClient, err := g.gatewaySelector.Next()
if err != nil {
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
}
res, err := gatewayClient.Stat(ctx, &storageprovider.StatRequest{Ref: &storageprovider.Reference{ResourceId: &driveItemID}})
switch {
case err != nil:
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
case res.Status.Code == cs3rpc.Code_CODE_OK:
// ok
case res.Status.Code == cs3rpc.Code_CODE_NOT_FOUND:
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, res.Status.Message)
return
case res.Status.Code == cs3rpc.Code_CODE_PERMISSION_DENIED:
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, res.Status.Message) // do not leak existence? check what graph does
return
case res.Status.Code == cs3rpc.Code_CODE_UNAUTHENTICATED:
errorcode.Unauthenticated.Render(w, r, http.StatusUnauthorized, res.Status.Message) // do not leak existence? check what graph does
return
default:
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, res.Status.Message)
return
}
driveItem, err := cs3ResourceToDriveItem(g.logger, res.Info)
if err != nil {
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
}
render.Status(r, http.StatusOK)
render.JSON(w, r, &driveItem)
}
// GetDriveItemChildren lists the children of a driveItem
func (g Graph) GetDriveItemChildren(w http.ResponseWriter, r *http.Request) {
g.logger.Info().Msg("Calling GetDriveItemChildren")
ctx := r.Context()
driveID, err := parseIDParam(r, "driveID")
if err != nil {
errorcode.RenderError(w, r, err)
return
}
driveItemID, err := parseIDParam(r, "driveItemID")
if err != nil {
errorcode.RenderError(w, r, err)
return
}
if driveID.StorageId != driveItemID.StorageId || driveID.SpaceId != driveItemID.SpaceId {
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, "Item does not exist")
return
}
/*
sanitizedPath := strings.TrimPrefix(r.URL.Path, "/graph/v1.0/")
// Parse the request with odata parser
odataReq, err := godata.ParseRequest(ctx, sanitizedPath, r.URL.Query())
if err != nil {
errorcode.InvalidRequest.Render(w, r, http.StatusBadRequest, err.Error())
return
}
*/
gatewayClient, err := g.gatewaySelector.Next()
if err != nil {
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
}
res, err := gatewayClient.ListContainer(ctx, &storageprovider.ListContainerRequest{
Ref: &storageprovider.Reference{ResourceId: &driveItemID},
})
switch {
case err != nil:
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
case res.Status.Code == cs3rpc.Code_CODE_OK:
// ok
case res.Status.Code == cs3rpc.Code_CODE_NOT_FOUND:
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, res.Status.Message)
return
case res.Status.Code == cs3rpc.Code_CODE_PERMISSION_DENIED:
errorcode.ItemNotFound.Render(w, r, http.StatusNotFound, res.Status.Message) // do not leak existence? check what graph does
return
case res.Status.Code == cs3rpc.Code_CODE_UNAUTHENTICATED:
errorcode.Unauthenticated.Render(w, r, http.StatusUnauthorized, res.Status.Message) // do not leak existence? check what graph does
return
default:
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, res.Status.Message)
return
}
files, err := formatDriveItems(g.logger, res.Infos)
if err != nil {
errorcode.GeneralException.Render(w, r, http.StatusInternalServerError, err.Error())
return
}
render.Status(r, http.StatusOK)
render.JSON(w, r, &ListResponse{Value: files})
}
func (g Graph) getDriveItem(ctx context.Context, ref storageprovider.Reference) (*libregraph.DriveItem, error) {
gatewayClient, err := g.gatewaySelector.Next()
if err != nil {
return nil, err
}
res, err := gatewayClient.Stat(ctx, &storageprovider.StatRequest{Ref: &ref})
if err != nil {
return nil, err
}
if res.Status.Code != cs3rpc.Code_CODE_OK {
refStr, _ := storagespace.FormatReference(&ref)
return nil, fmt.Errorf("could not stat %s: %s", refStr, res.Status.Message)
}
return cs3ResourceToDriveItem(g.logger, res.Info)
}
func (g Graph) getRemoteItem(ctx context.Context, root *storageprovider.ResourceId, baseURL *url.URL) (*libregraph.RemoteItem, error) {
gatewayClient, err := g.gatewaySelector.Next()
if err != nil {
return nil, err
}
ref := &storageprovider.Reference{
ResourceId: root,
}
res, err := gatewayClient.Stat(ctx, &storageprovider.StatRequest{Ref: ref})
if err != nil {
return nil, err
}
if res.Status.Code != cs3rpc.Code_CODE_OK {
// Only log this, there could be mountpoints which have no grant
g.logger.Debug().Msg(res.Status.Message)
return nil, errors.New("could not fetch grant resource for the mountpoint")
}
item, err := cs3ResourceToRemoteItem(res.Info)
if err != nil {
return nil, err
}
if baseURL != nil && res.GetInfo() != nil && res.GetInfo().GetSpace() != nil {
// TODO read from StorageSpace ... needs Opaque for now
// TODO how do we build the url?
// for now: read from request
item.Name = libregraph.PtrString(res.GetInfo().GetName())
if res.GetInfo().GetSpace().GetRoot() != nil {
webDavURL := *baseURL
relativePath := res.GetInfo().GetPath()
webDavURL.Path = path.Join(webDavURL.Path, storagespace.FormatResourceID(*res.GetInfo().GetSpace().GetRoot()), relativePath)
item.WebDavUrl = libregraph.PtrString(webDavURL.String())
}
}
return item, nil
}
func formatDriveItems(logger *log.Logger, mds []*storageprovider.ResourceInfo) ([]*libregraph.DriveItem, error) {
responses := make([]*libregraph.DriveItem, 0, len(mds))
for i := range mds {
res, err := cs3ResourceToDriveItem(logger, mds[i])
if err != nil {
return nil, err
}
responses = append(responses, res)
}
return responses, nil
}
func cs3TimestampToTime(t *types.Timestamp) time.Time {
return time.Unix(int64(t.Seconds), int64(t.Nanos))
}
func cs3ResourceToDriveItem(logger *log.Logger, res *storageprovider.ResourceInfo) (*libregraph.DriveItem, error) {
size := new(int64)
*size = int64(res.Size) // TODO lurking overflow: make size of libregraph drive item use uint64
driveItem := &libregraph.DriveItem{
Id: libregraph.PtrString(storagespace.FormatResourceID(*res.Id)),
Size: size,
}
if name := path.Base(res.Path); name != "" {
driveItem.Name = &name
}
if res.Etag != "" {
driveItem.ETag = &res.Etag
}
if res.Mtime != nil {
lastModified := cs3TimestampToTime(res.Mtime)
driveItem.LastModifiedDateTime = &lastModified
}
if res.Type == storageprovider.ResourceType_RESOURCE_TYPE_FILE && res.MimeType != "" {
// We cannot use a libregraph.File here because the openapi codegenerator autodetects 'File' as a go type ...
driveItem.File = &libregraph.OpenGraphFile{
MimeType: &res.MimeType,
}
}
if res.Type == storageprovider.ResourceType_RESOURCE_TYPE_CONTAINER {
driveItem.Folder = &libregraph.Folder{}
}
driveItem.Audio = cs3ResourceToDriveItemAudioFacet(logger, res)
return driveItem, nil
}
func cs3ResourceToDriveItemAudioFacet(logger *log.Logger, res *storageprovider.ResourceInfo) *libregraph.Audio {
if !strings.HasPrefix(res.MimeType, "audio/") {
return nil
}
k := res.ArbitraryMetadata.Metadata
if k == nil {
return nil
}
var audio = &libregraph.Audio{}
if ok := unmarshalStringMap(logger, audio, k, "libre.graph.audio."); ok {
return audio
}
return nil
}
func getFieldName(structField reflect.StructField) string {
tag := structField.Tag.Get("json")
if tag == "" {
return structField.Name
}
return strings.Split(tag, ",")[0]
}
func unmarshalStringMap(logger *log.Logger, out any, flatMap map[string]string, prefix string) bool {
nonEmpty := false
obj := reflect.ValueOf(out).Elem()
for i := 0; i < obj.NumField(); i++ {
field := obj.Field(i)
structField := obj.Type().Field(i)
mapKey := prefix + getFieldName(structField)
if value, ok := flatMap[mapKey]; ok {
if field.Kind() == reflect.Ptr {
newValue := reflect.New(field.Type().Elem())
var tmp any
var err error
switch t := newValue.Type().Elem().Kind(); t {
case reflect.String:
tmp = value
case reflect.Int32:
tmp, err = strconv.ParseInt(value, 10, 32)
case reflect.Int64:
tmp, err = strconv.ParseInt(value, 10, 64)
case reflect.Bool:
tmp, err = strconv.ParseBool(value)
default:
err = errors.New("unsupported type")
logger.Error().Err(err).Str("type", t.String()).Str("mapKey", mapKey).Msg("target field type for value of mapKey is not supported")
}
if err != nil {
logger.Error().Err(err).Str("mapKey", mapKey).Msg("unmarshalling failed")
continue
}
newValue.Elem().Set(reflect.ValueOf(tmp).Convert(field.Type().Elem()))
field.Set(newValue)
nonEmpty = true
}
}
}
return nonEmpty
}
func cs3ResourceToRemoteItem(res *storageprovider.ResourceInfo) (*libregraph.RemoteItem, error) {
size := new(int64)
*size = int64(res.Size) // TODO lurking overflow: make size of libregraph drive item use uint64
remoteItem := &libregraph.RemoteItem{
Id: libregraph.PtrString(storagespace.FormatResourceID(*res.Id)),
Size: size,
}
if res.GetPath() != "" {
remoteItem.Path = libregraph.PtrString(path.Clean(res.GetPath()))
}
if res.Etag != "" {
remoteItem.ETag = &res.Etag
}
if res.Mtime != nil {
lastModified := cs3TimestampToTime(res.Mtime)
remoteItem.LastModifiedDateTime = &lastModified
}
if res.Type == storageprovider.ResourceType_RESOURCE_TYPE_FILE && res.MimeType != "" {
// We cannot use a libregraph.File here because the openapi codegenerator autodetects 'File' as a go type ...
remoteItem.File = &libregraph.OpenGraphFile{
MimeType: &res.MimeType,
}
}
if res.Type == storageprovider.ResourceType_RESOURCE_TYPE_CONTAINER {
remoteItem.Folder = &libregraph.Folder{}
}
if res.GetSpace() != nil && res.GetSpace().GetRoot() != nil {
remoteItem.RootId = libregraph.PtrString(storagespace.FormatResourceID(*res.GetSpace().GetRoot()))
grantSpaceAlias := utils.ReadPlainFromOpaque(res.GetSpace().GetOpaque(), "spaceAlias")
if grantSpaceAlias != "" {
remoteItem.DriveAlias = libregraph.PtrString(grantSpaceAlias)
}
}
return remoteItem, nil
}
func (g Graph) getPathForResource(ctx context.Context, id storageprovider.ResourceId) (string, error) {
gatewayClient, err := g.gatewaySelector.Next()
if err != nil {
return "", err
}
res, err := gatewayClient.GetPath(ctx, &storageprovider.GetPathRequest{ResourceId: &id})
if err != nil {
return "", err
}
if res.Status.Code != cs3rpc.Code_CODE_OK {
return "", fmt.Errorf("could not stat %v: %s", id, res.Status.Message)
}
return res.Path, err
}
// getSpecialDriveItems reads properties from the opaque and transforms them into driveItems
func (g Graph) getSpecialDriveItems(ctx context.Context, baseURL *url.URL, space *storageprovider.StorageSpace) []libregraph.DriveItem {
if space.GetRoot().GetStorageId() == utils.ShareStorageProviderID {
return nil // no point in stating the ShareStorageProvider
}
if space.Opaque == nil {
return nil
}
imageNode := utils.ReadPlainFromOpaque(space.Opaque, SpaceImageSpecialFolderName)
readmeNode := utils.ReadPlainFromOpaque(space.Opaque, ReadmeSpecialFolderName)
cachekey := spaceRootStatKey(space.Root, imageNode, readmeNode)
// if the root is older or equal to our cache we can reuse the cached extended spaces properties
if entry := g.specialDriveItemsCache.Get(cachekey); entry != nil {
if cached, ok := entry.Value().(specialDriveItemEntry); ok {
if cached.rootMtime != nil && space.Mtime != nil {
// beware, LaterTS does not handle equalness. it returns t1 if t1 > t2, else t2, so a >= check looks like this
if utils.LaterTS(space.Mtime, cached.rootMtime) == cached.rootMtime {
return cached.specialDriveItems
}
}
}
}
var spaceItems []libregraph.DriveItem
spaceItems = g.fetchSpecialDriveItem(ctx, spaceItems, SpaceImageSpecialFolderName, imageNode, space, baseURL)
spaceItems = g.fetchSpecialDriveItem(ctx, spaceItems, ReadmeSpecialFolderName, readmeNode, space, baseURL)
// cache properties
spacePropertiesEntry := specialDriveItemEntry{
specialDriveItems: spaceItems,
rootMtime: space.Mtime,
}
g.specialDriveItemsCache.Set(cachekey, spacePropertiesEntry, time.Duration(g.config.Spaces.ExtendedSpacePropertiesCacheTTL))
return spaceItems
}
func (g Graph) fetchSpecialDriveItem(ctx context.Context, spaceItems []libregraph.DriveItem, itemName string, itemNode string, space *storageprovider.StorageSpace, baseURL *url.URL) []libregraph.DriveItem {
var ref storageprovider.Reference
if itemNode != "" {
rid, _ := storagespace.ParseID(itemNode)
rid.StorageId = space.GetRoot().GetStorageId()
ref = storageprovider.Reference{
ResourceId: &rid,
}
spaceItem := g.getSpecialDriveItem(ctx, ref, itemName, baseURL, space)
if spaceItem != nil {
spaceItems = append(spaceItems, *spaceItem)
}
}
return spaceItems
}
// generates a space root stat cache key used to detect changes in a space
// takes into account the special nodes because changing metadata does not affect the etag / mtime
func spaceRootStatKey(id *storageprovider.ResourceId, imagenode, readmeNode string) string {
if id == nil {
return ""
}
sha3 := sha3.NewShake256()
_, _ = sha3.Write([]byte(id.GetStorageId()))
_, _ = sha3.Write([]byte(id.GetSpaceId()))
_, _ = sha3.Write([]byte(id.GetOpaqueId()))
_, _ = sha3.Write([]byte(imagenode))
_, _ = sha3.Write([]byte(readmeNode))
h := make([]byte, 64)
_, _ = sha3.Read(h)
return fmt.Sprintf("%x", h)
}
type specialDriveItemEntry struct {
specialDriveItems []libregraph.DriveItem
rootMtime *types.Timestamp
}
func (g Graph) getSpecialDriveItem(ctx context.Context, ref storageprovider.Reference, itemName string, baseURL *url.URL, space *storageprovider.StorageSpace) *libregraph.DriveItem {
var spaceItem *libregraph.DriveItem
if ref.GetResourceId().GetSpaceId() == "" && ref.GetResourceId().GetOpaqueId() == "" {
return nil
}
// FIXME we should send a fieldmask 'path' and return it as the Path property to save an additional call to the storage.
// To do that we need to align the useg of ResourceInfo.Name vs ResourceInfo.Path. By default, only the name should be set
// and Path should always be relative to the space root OR the resource the current user can access ...
spaceItem, err := g.getDriveItem(ctx, ref)
if err != nil {
g.logger.Debug().Err(err).Str("ID", ref.GetResourceId().GetOpaqueId()).Str("name", itemName).Msg("Could not get item info")
return nil
}
itemPath := ref.Path
if itemPath == "" {
// lookup by id
itemPath, err = g.getPathForResource(ctx, *ref.ResourceId)
if err != nil {
g.logger.Debug().Err(err).Str("ID", ref.GetResourceId().GetOpaqueId()).Str("name", itemName).Msg("Could not get item path")
return nil
}
}
spaceItem.SpecialFolder = &libregraph.SpecialFolder{Name: libregraph.PtrString(itemName)}
webdavURL := *baseURL
webdavURL.Path = path.Join(webdavURL.Path, space.Id.OpaqueId, itemPath)
spaceItem.WebDavUrl = libregraph.PtrString(webdavURL.String())
return spaceItem
}