@@ -20,10 +20,12 @@ import (
2020 "os/exec"
2121 "path"
2222 "path/filepath"
23+ "regexp"
2324 "strings"
25+ "time"
2426
25- "k8s.io/apimachinery/pkg/util/sets"
2627 "k8s.io/klog"
28+ pathutils "k8s.io/utils/path"
2729)
2830
2931const (
@@ -46,6 +48,14 @@ const (
4648 // 'fsck' found errors but exited without correcting them
4749 fsckErrorsUncorrected = 4
4850 defaultMountCommand = "mount"
51+ // scsi_id output should be in the form of:
52+ // 0Google PersistentDisk <disk name>
53+ scsiPattern = `^0Google\s+PersistentDisk\s+([\S]+)\s*$`
54+ )
55+
56+ var (
57+ // regex to parse scsi_id output and extract the serial
58+ scsiRegex = regexp .MustCompile (scsiPattern )
4959)
5060
5161// DeviceUtils are a collection of methods that act on the devices attached
@@ -57,7 +67,7 @@ type DeviceUtils interface {
5767
5868 // VerifyDevicePath returns the first of the list of device paths that
5969 // exists on the machine, or an empty string if none exists
60- VerifyDevicePath (devicePaths []string ) (string , error )
70+ VerifyDevicePath (devicePaths []string , diskName string ) (string , error )
6171}
6272
6373type deviceUtils struct {
@@ -85,75 +95,144 @@ func (m *deviceUtils) GetDiskByIdPaths(deviceName string, partition string) []st
8595 return devicePaths
8696}
8797
88- // Returns the first path that exists, or empty string if none exist.
89- func (m * deviceUtils ) VerifyDevicePath (devicePaths []string ) (string , error ) {
90- sdBefore , err := filepath .Glob (diskSDPattern )
91- if err != nil {
92- // Seeing this error means that the diskSDPattern is malformed.
93- klog .Errorf ("Error filepath.Glob(\" %s\" ): %v\r \n " , diskSDPattern , err )
94- }
95- sdBeforeSet := sets .NewString (sdBefore ... )
96- // TODO(#69): Verify udevadm works as intended in driver
97- if err := udevadmChangeToNewDrives (sdBeforeSet ); err != nil {
98- // udevadm errors should not block disk detachment, log and continue
99- klog .Errorf ("udevadmChangeToNewDrives failed with: %v" , err )
100- }
101-
102- for _ , path := range devicePaths {
103- if pathExists , err := pathExists (path ); err != nil {
104- return "" , fmt .Errorf ("Error checking if path exists: %v" , err )
98+ func existingDevicePath (devicePaths []string ) (string , error ) {
99+ for _ , devicePath := range devicePaths {
100+ if pathExists , err := pathExists (devicePath ); err != nil {
101+ return "" , fmt .Errorf ("error checking if path exists: %v" , err )
105102 } else if pathExists {
106- return path , nil
103+ return devicePath , nil
107104 }
108105 }
109-
110106 return "" , nil
111107}
112108
113- // Triggers the application of udev rules by calling "udevadm trigger
114- // --action=change" for newly created "/dev/sd*" drives (exist only in
115- // after set). This is workaround for Issue #7972. Once the underlying
116- // issue has been resolved, this may be removed.
117- func udevadmChangeToNewDrives (sdBeforeSet sets.String ) error {
118- sdAfter , err := filepath .Glob (diskSDPattern )
109+ // getScsiSerial assumes that /lib/udev/scsi_id exists and will error if it
110+ // doesnt. It is the callers responsibility to verify the existence of this
111+ // tool. Calls scsi_id on the given devicePath to get the serial number reported
112+ // by that device.
113+ func getScsiSerial (devicePath string ) (string , error ) {
114+ out , err := exec .Command (
115+ "/lib/udev_containerized/scsi_id" ,
116+ "--page=0x83" ,
117+ "--whitelisted" ,
118+ fmt .Sprintf ("--device=%v" , devicePath )).CombinedOutput ()
119119 if err != nil {
120- return fmt .Errorf ("Error filepath.Glob( \" %s \" ) : %v\r \n " , diskSDPattern , err )
120+ return "" , fmt .Errorf ("scsi_id failed for device %q with output %s : %v" , devicePath , string ( out ) , err )
121121 }
122122
123- for _ , sd := range sdAfter {
124- if ! sdBeforeSet .Has (sd ) {
125- return udevadmChangeToDrive (sd )
126- }
123+ return parseScsiSerial (string (out ))
124+ }
125+
126+ // Parse the output returned by scsi_id and extract the serial number
127+ func parseScsiSerial (output string ) (string , error ) {
128+ substrings := scsiRegex .FindStringSubmatch (output )
129+ if substrings == nil {
130+ return "" , fmt .Errorf ("scsi_id output cannot be parsed: %q" , output )
127131 }
128132
129- return nil
133+ return substrings [ 1 ], nil
130134}
131135
132- // Calls "udevadm trigger --action=change" on the specified drive.
133- // drivePath must be the block device path to trigger on, in the format "/dev/sd*", or a symlink to it.
134- // This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed.
135- func udevadmChangeToDrive (drivePath string ) error {
136- klog .V (4 ).Infof ("udevadmChangeToDrive: drive=%q" , drivePath )
136+ // VerifyDevicePath returns the first devicePath that maps to a real disk in the
137+ // candidate devicePaths or an empty string if none is found. If
138+ // /lib/udev/scsi_id exists it will attempt to fix any issues caused by missing
139+ // paths or mismatched devices by running a udevadm --trigger.
140+ func (m * deviceUtils ) VerifyDevicePath (devicePaths []string , diskName string ) (string , error ) {
141+ devicePath , err := existingDevicePath (devicePaths )
142+ if err != nil {
143+ return "" , fmt .Errorf ("failed to check for existing device path: %v" , err )
144+ }
137145
138- // Evaluate symlink, if any
139- drive , err := filepath . EvalSymlinks ( drivePath )
146+ scsiIDPath := "/lib/udev_containerized/scsi_id"
147+ exists , err := pathutils . Exists ( pathutils . CheckFollowSymlink , scsiIDPath )
140148 if err != nil {
141- return fmt .Errorf ("udevadmChangeToDrive: filepath.EvalSymlinks(%q) failed with %v." , drivePath , err )
149+ return "" , fmt .Errorf ("failed to check scsi_id existence: %v" , err )
150+ }
151+ if ! exists {
152+ // No SCSI ID tool, no udevadm fixing is possible so just return best
153+ // effort devicePath.
154+ klog .Warningf ("Could not find scsi_id tool at %s, no udevadm fixing is possible so we just return the best effort devicePath" , scsiIDPath )
155+ return devicePath , nil
156+ }
157+
158+ if len (devicePath ) == 0 {
159+ // Couldn't find the path so we need to find a /dev/sdx with the SCSI
160+ // serial that matches diskName. Then we run udevadm trigger on that
161+ // device to get the device to show up in /dev/by-id/
162+ err := udevadmTriggerForDiskIfExists (diskName )
163+ if err != nil {
164+ return "" , fmt .Errorf ("failed to trigger udevadm fix for disk %s: %v" , diskName , err )
165+ }
166+ // Try get the deviceName again after potentially fixing it with the
167+ // udev command
168+ return existingDevicePath (devicePaths )
169+ }
170+
171+ // If there exists a devicePath we make sure disk at /dev/sdx matches the
172+ // expected disk at devicePath by matching SCSI Serial to the disk name
173+ devSDX , err := filepath .EvalSymlinks (devicePath )
174+ if err != nil {
175+ return "" , fmt .Errorf ("filepath.EvalSymlinks(%q) failed with %v." , devicePath , err )
142176 }
143177 // Check to make sure input is "/dev/sd*"
144- if ! strings .Contains (drive , diskSDPath ) {
145- return fmt .Errorf ("udevadmChangeToDrive: expected input in the form \" %s\" but drive is %q." , diskSDPattern , drive )
178+ if ! strings .Contains (devSDX , diskSDPath ) {
179+ return "" , fmt .Errorf ("expected input in the form \" %s\" but drive is %q." , diskSDPattern , devSDX )
180+ }
181+ scsiSerial , err := getScsiSerial (devSDX )
182+ if err != nil {
183+ return "" , fmt .Errorf ("couldn't get SCSI serial number for disk %s: %v" , diskName , err )
146184 }
185+ if scsiSerial != diskName {
186+ err := udevadmTriggerForDiskIfExists (diskName )
187+ if err != nil {
188+ return "" , fmt .Errorf ("failed to trigger udevadm fix for disk %s: %v" , diskName , err )
189+ }
190+ }
191+ return devicePath , nil
192+ }
147193
194+ func udevadmTriggerForDiskIfExists (diskName string ) error {
195+ sds , err := filepath .Glob (diskSDPattern )
196+ if err != nil {
197+ return fmt .Errorf ("failed to filepath.Glob(\" %s\" ): %v" , diskSDPattern , err )
198+ }
199+ for _ , devSDX := range sds {
200+ scsiSerial , err := getScsiSerial (devSDX )
201+ if err != nil {
202+ return fmt .Errorf ("failed to get SCSI Serial num for %s: %v" , devSDX , err )
203+ }
204+ if scsiSerial == diskName {
205+ // Found the disk that we're looking for so run a trigger on it
206+ // to resolve its /dev/by-id/ path
207+ klog .Warningf ("udevadm --trigger running to fix disk at path %s which has SCSI ID %s" , devSDX , scsiSerial )
208+ err := udevadmChangeToDrive (devSDX )
209+ if err != nil {
210+ return fmt .Errorf ("failed to fix disk at path %s which has SCSI ID %s: %v" , devSDX , scsiSerial , err )
211+ }
212+ return nil
213+ }
214+ }
215+ klog .Warningf ("udevadm --trigger requested to fix disk %s but no such disk was found in %v" , diskName , sds )
216+ return nil
217+ }
218+
219+ // Calls "udevadm trigger --action=change" on the specified drive.
220+ // drivePath must be the block device path to trigger on, in the format "/dev/sd*", or a symlink to it.
221+ // This is workaround for Issue #7972. Once the underlying issue has been resolved, this may be removed.
222+ func udevadmChangeToDrive (devSDX string ) error {
148223 // Call "udevadm trigger --action=change --property-match=DEVNAME=/dev/sd..."
149- _ , err = exec .Command (
224+ out , err : =exec .Command (
150225 "udevadm" ,
151226 "trigger" ,
152227 "--action=change" ,
153- fmt .Sprintf ("--property-match=DEVNAME=%s" , drive )).CombinedOutput ()
228+ fmt .Sprintf ("--property-match=DEVNAME=%s" , devSDX )).CombinedOutput ()
154229 if err != nil {
155- return fmt .Errorf ("udevadmChangeToDrive: udevadm trigger failed for drive %q with % v." , drive , err )
230+ return fmt .Errorf ("udevadmChangeToDrive: udevadm trigger failed for drive %q with output %s: % v." , devSDX , string ( out ) , err )
156231 }
232+ // udevadm takes a little bit to work its magic in the background - since
233+ // this fix only gets called when something is wrong and we *need* the
234+ // udevadm we wait a second or two before returning a successful udevadm
235+ time .Sleep (time .Second )
157236 return nil
158237}
159238
0 commit comments