Skip to content

Commit 8369041

Browse files
sagor999sagor999
committed
refactor dispose workspace when using pvc
1 parent e8bf53b commit 8369041

File tree

2 files changed

+99
-82
lines changed

2 files changed

+99
-82
lines changed

‎components/ws-manager/pkg/manager/manager.go

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ import (
1717

1818
validation "github.com/go-ozzo/ozzo-validation"
1919
"github.com/opentracing/opentracing-go"
20+
"github.com/sirupsen/logrus"
2021
"golang.org/x/xerrors"
2122
"google.golang.org/grpc"
2223
"google.golang.org/grpc/codes"
@@ -29,12 +30,15 @@ import (
2930
corev1 "k8s.io/api/core/v1"
3031
k8serr "k8s.io/apimachinery/pkg/api/errors"
3132
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
33+
"k8s.io/apimachinery/pkg/fields"
3234
"k8s.io/apimachinery/pkg/labels"
3335
"k8s.io/apimachinery/pkg/runtime"
3436
"k8s.io/apimachinery/pkg/types"
3537
"k8s.io/apimachinery/pkg/util/wait"
38+
"k8s.io/apimachinery/pkg/watch"
3639
"k8s.io/client-go/kubernetes"
3740
covev1client "k8s.io/client-go/kubernetes/typed/core/v1"
41+
"k8s.io/client-go/tools/cache"
3842
"k8s.io/client-go/tools/record"
3943
"k8s.io/client-go/util/retry"
4044
"sigs.k8s.io/controller-runtime/pkg/client"
@@ -55,6 +59,7 @@ import (
5559

5660
volumesnapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/apis/volumesnapshot/v1"
5761
volumesnapshotclientv1 "github.com/kubernetes-csi/external-snapshotter/client/v4/clientset/versioned"
62+
watchtools "k8s.io/client-go/tools/watch"
5863
)
5964

6065
// Manager is a kubernetes backed implementation of a workspace manager
@@ -1522,6 +1527,96 @@ func (m *Manager) connectToWorkspaceDaemon(ctx context.Context, wso workspaceObj
15221527
return wsdaemon.NewWorkspaceContentServiceClient(conn), nil
15231528
}
15241529

1530+
func (m *Manager) createWorkspaceSnapshotFromPVC(ctx context.Context, pvcName string, pvcVolumeSnapshotName string, pvcVolumeSnapshotClassName string, workspaceID string, labels map[string]string) error {
1531+
// create snapshot object out of PVC
1532+
volumeSnapshot := &volumesnapshotv1.VolumeSnapshot{
1533+
ObjectMeta: metav1.ObjectMeta{
1534+
Name: pvcVolumeSnapshotName,
1535+
Namespace: m.Config.Namespace,
1536+
Annotations: map[string]string{workspaceIDAnnotation: workspaceID},
1537+
Labels: labels,
1538+
},
1539+
Spec: volumesnapshotv1.VolumeSnapshotSpec{
1540+
Source: volumesnapshotv1.VolumeSnapshotSource{
1541+
PersistentVolumeClaimName: &pvcName,
1542+
},
1543+
VolumeSnapshotClassName: &pvcVolumeSnapshotClassName,
1544+
},
1545+
}
1546+
1547+
err := m.Clientset.Create(ctx, volumeSnapshot)
1548+
if err != nil && !k8serr.IsAlreadyExists(err) {
1549+
err = xerrors.Errorf("cannot create volumesnapshot: %v", err)
1550+
return err
1551+
}
1552+
return nil
1553+
}
1554+
1555+
func (m *Manager) waitForWorkspaceVolumeSnapshotReady(ctx context.Context, pvcVolumeSnapshotName string, log *logrus.Entry) (pvcVolumeSnapshotContentName string, readyVolumeSnapshot bool, err error) {
1556+
log = log.WithField("VolumeSnapshot.Name", pvcVolumeSnapshotName)
1557+
1558+
var volumeSnapshotWatcher *watchtools.RetryWatcher
1559+
volumeSnapshotWatcher, err = watchtools.NewRetryWatcher("1", &cache.ListWatch{
1560+
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
1561+
return m.VolumeSnapshotClient.SnapshotV1().VolumeSnapshots(m.Config.Namespace).Watch(ctx, metav1.ListOptions{
1562+
FieldSelector: fields.OneTermEqualSelector("metadata.name", pvcVolumeSnapshotName).String(),
1563+
})
1564+
},
1565+
})
1566+
if err != nil {
1567+
log.WithError(err).Info("fall back to exponential backoff retry")
1568+
// we can not create a retry watcher, we fall back to exponential backoff retry
1569+
backoff := wait.Backoff{
1570+
Steps: 30,
1571+
Duration: 100 * time.Millisecond,
1572+
Factor: 1.5,
1573+
Jitter: 0.1,
1574+
Cap: 10 * time.Minute,
1575+
}
1576+
err = wait.ExponentialBackoff(backoff, func() (bool, error) {
1577+
var vs volumesnapshotv1.VolumeSnapshot
1578+
err := m.Clientset.Get(ctx, types.NamespacedName{Namespace: m.Config.Namespace, Name: pvcVolumeSnapshotName}, &vs)
1579+
if err != nil {
1580+
if k8serr.IsNotFound(err) {
1581+
// volumesnapshot doesn't exist yet, retry again
1582+
return false, nil
1583+
}
1584+
log.WithError(err).Error("was unable to get volume snapshot")
1585+
return false, err
1586+
}
1587+
if vs.Status != nil && vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse && vs.Status.BoundVolumeSnapshotContentName != nil {
1588+
pvcVolumeSnapshotContentName = *vs.Status.BoundVolumeSnapshotContentName
1589+
return true, nil
1590+
}
1591+
return false, nil
1592+
})
1593+
if err != nil {
1594+
log.WithError(err).Errorf("failed while waiting for volume snapshot to get ready")
1595+
return "", false, err
1596+
}
1597+
readyVolumeSnapshot = true
1598+
} else {
1599+
for event := range volumeSnapshotWatcher.ResultChan() {
1600+
vs, ok := event.Object.(*volumesnapshotv1.VolumeSnapshot)
1601+
if !ok {
1602+
log.Errorf("unexpected type assertion %T", event.Object)
1603+
continue
1604+
}
1605+
1606+
if vs != nil && vs.Status != nil && vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse && vs.Status.BoundVolumeSnapshotContentName != nil {
1607+
pvcVolumeSnapshotContentName = *vs.Status.BoundVolumeSnapshotContentName
1608+
readyVolumeSnapshot = true
1609+
break
1610+
}
1611+
}
1612+
1613+
// stop the volume snapshot retry watcher
1614+
volumeSnapshotWatcher.Stop()
1615+
}
1616+
1617+
return pvcVolumeSnapshotContentName, readyVolumeSnapshot, nil
1618+
}
1619+
15251620
// newWssyncConnectionFactory creates a new wsdaemon connection factory based on the wsmanager configuration
15261621
func newWssyncConnectionFactory(managerConfig config.Configuration) (grpcpool.Factory, error) {
15271622
cfg := managerConfig.WorkspaceDaemon

‎components/ws-manager/pkg/manager/monitor.go

Lines changed: 4 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,10 @@ import (
2626

2727
corev1 "k8s.io/api/core/v1"
2828
k8serr "k8s.io/apimachinery/pkg/api/errors"
29-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30-
"k8s.io/apimachinery/pkg/fields"
3129
"k8s.io/apimachinery/pkg/labels"
3230
"k8s.io/apimachinery/pkg/types"
33-
"k8s.io/apimachinery/pkg/util/wait"
3431
"k8s.io/apimachinery/pkg/watch"
35-
"k8s.io/client-go/tools/cache"
3632
"k8s.io/client-go/tools/record"
37-
watchtools "k8s.io/client-go/tools/watch"
3833
"sigs.k8s.io/controller-runtime/pkg/client"
3934

4035
"github.com/gitpod-io/gitpod/common-go/kubernetes"
@@ -1115,90 +1110,17 @@ func (m *Monitor) finalizeWorkspaceContent(ctx context.Context, wso *workspaceOb
11151110
// pvc was created with the name of the pod. see createDefiniteWorkspacePod()
11161111
pvcName := wso.Pod.Name
11171112
if !createdVolumeSnapshot {
1118-
// create snapshot object out of PVC
1119-
volumeSnapshot := &volumesnapshotv1.VolumeSnapshot{
1120-
ObjectMeta: metav1.ObjectMeta{
1121-
Name: pvcVolumeSnapshotName,
1122-
Namespace: m.manager.Config.Namespace,
1123-
Annotations: map[string]string{workspaceIDAnnotation: workspaceID},
1124-
Labels: wso.Pod.Labels,
1125-
},
1126-
Spec: volumesnapshotv1.VolumeSnapshotSpec{
1127-
Source: volumesnapshotv1.VolumeSnapshotSource{
1128-
PersistentVolumeClaimName: &pvcName,
1129-
},
1130-
VolumeSnapshotClassName: &pvcVolumeSnapshotClassName,
1131-
},
1132-
}
1133-
1134-
err = m.manager.Clientset.Create(ctx, volumeSnapshot)
1135-
if err != nil && !k8serr.IsAlreadyExists(err) {
1136-
err = xerrors.Errorf("cannot create volumesnapshot: %v", err)
1113+
err = m.manager.createWorkspaceSnapshotFromPVC(ctx, pvcName, pvcVolumeSnapshotName, pvcVolumeSnapshotClassName, workspaceID, wso.Pod.Labels)
1114+
if err != nil {
11371115
return nil, err
11381116
}
11391117
createdVolumeSnapshot = true
11401118
volumeSnapshotTime = time.Now()
11411119
}
11421120
if createdVolumeSnapshot {
1143-
log = log.WithField("VolumeSnapshot.Name", pvcVolumeSnapshotName)
1144-
1145-
var volumeSnapshotWatcher *watchtools.RetryWatcher
1146-
volumeSnapshotWatcher, err = watchtools.NewRetryWatcher("1", &cache.ListWatch{
1147-
WatchFunc: func(options metav1.ListOptions) (watch.Interface, error) {
1148-
return m.manager.VolumeSnapshotClient.SnapshotV1().VolumeSnapshots(m.manager.Config.Namespace).Watch(ctx, metav1.ListOptions{
1149-
FieldSelector: fields.OneTermEqualSelector("metadata.name", pvcVolumeSnapshotName).String(),
1150-
})
1151-
},
1152-
})
1121+
pvcVolumeSnapshotContentName, readyVolumeSnapshot, err = m.manager.waitForWorkspaceVolumeSnapshotReady(ctx, pvcVolumeSnapshotName, log)
11531122
if err != nil {
1154-
log.WithError(err).Info("fall back to exponential backoff retry")
1155-
// we can not create a retry watcher, we fall back to exponential backoff retry
1156-
backoff := wait.Backoff{
1157-
Steps: 30,
1158-
Duration: 100 * time.Millisecond,
1159-
Factor: 1.5,
1160-
Jitter: 0.1,
1161-
Cap: 10 * time.Minute,
1162-
}
1163-
err = wait.ExponentialBackoff(backoff, func() (bool, error) {
1164-
var vs volumesnapshotv1.VolumeSnapshot
1165-
err := m.manager.Clientset.Get(ctx, types.NamespacedName{Namespace: m.manager.Config.Namespace, Name: pvcVolumeSnapshotName}, &vs)
1166-
if err != nil {
1167-
if k8serr.IsNotFound(err) {
1168-
// volumesnapshot doesn't exist yet, retry again
1169-
return false, nil
1170-
}
1171-
log.WithError(err).Error("was unable to get volume snapshot")
1172-
return false, err
1173-
}
1174-
if vs.Status != nil && vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse && vs.Status.BoundVolumeSnapshotContentName != nil {
1175-
pvcVolumeSnapshotContentName = *vs.Status.BoundVolumeSnapshotContentName
1176-
return true, nil
1177-
}
1178-
return false, nil
1179-
})
1180-
if err != nil {
1181-
log.WithError(err).Errorf("failed while waiting for volume snapshot to get ready")
1182-
return nil, err
1183-
}
1184-
readyVolumeSnapshot = true
1185-
} else {
1186-
for event := range volumeSnapshotWatcher.ResultChan() {
1187-
vs, ok := event.Object.(*volumesnapshotv1.VolumeSnapshot)
1188-
if !ok {
1189-
log.Errorf("unexpected type assertion %T", event.Object)
1190-
continue
1191-
}
1192-
1193-
if vs != nil && vs.Status != nil && vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse && vs.Status.BoundVolumeSnapshotContentName != nil {
1194-
pvcVolumeSnapshotContentName = *vs.Status.BoundVolumeSnapshotContentName
1195-
readyVolumeSnapshot = true
1196-
break
1197-
}
1198-
}
1199-
1200-
// stop the volume snapshot retry watcher
1201-
volumeSnapshotWatcher.Stop()
1123+
return nil, err
12021124
}
12031125

12041126
hist, err := m.manager.metrics.volumeSnapshotTimeHistVec.GetMetricWithLabelValues(wsType, wso.Pod.Labels[workspaceClassLabel])

0 commit comments

Comments
 (0)