1
1
import * as debugLib from 'debug' ;
2
+ import * as pathLib from 'path' ;
3
+ const sortBy = require ( 'lodash.sortby' ) ;
4
+ const groupBy = require ( 'lodash.groupby' ) ;
2
5
3
6
import {
4
7
EntityToFix ,
8
+ FixChangesSummary ,
5
9
FixOptions ,
6
- WithFixChangesApplied ,
10
+ RemediationChanges ,
11
+ Workspace ,
7
12
} from '../../../../types' ;
8
13
import { PluginFixResponse } from '../../../types' ;
9
14
import { updateDependencies } from './update-dependencies' ;
10
15
import { MissingRemediationDataError } from '../../../../lib/errors/missing-remediation-data' ;
11
16
import { MissingFileNameError } from '../../../../lib/errors/missing-file-name' ;
12
17
import { partitionByFixable } from './is-supported' ;
13
18
import { NoFixesCouldBeAppliedError } from '../../../../lib/errors/no-fixes-applied' ;
14
- import { parseRequirementsFile } from './update-dependencies/requirements-file-parser' ;
19
+ import { extractProvenance } from './extract-version-provenance' ;
20
+ import {
21
+ ParsedRequirements ,
22
+ parseRequirementsFile ,
23
+ } from './update-dependencies/requirements-file-parser' ;
15
24
16
25
const debug = debugLib ( 'snyk-fix:python:requirements.txt' ) ;
17
26
@@ -26,56 +35,207 @@ export async function pipRequirementsTxt(
26
35
skipped : [ ] ,
27
36
} ;
28
37
29
- const { fixable, skipped } = await partitionByFixable ( entities ) ;
30
- handlerResult . skipped . push ( ...skipped ) ;
38
+ const { fixable, skipped : notFixable } = await partitionByFixable ( entities ) ;
39
+ handlerResult . skipped . push ( ...notFixable ) ;
31
40
32
- for ( const entity of fixable ) {
33
- try {
34
- const fixedEntity = await fixIndividualRequirementsTxt ( entity , options ) ;
35
- handlerResult . succeeded . push ( fixedEntity ) ;
36
- } catch ( e ) {
37
- handlerResult . failed . push ( { original : entity , error : e } ) ;
38
- }
41
+ const ordered = sortByDirectory ( fixable ) ;
42
+ const fixedFilesCache : string [ ] = [ ] ;
43
+ for ( const dir of Object . keys ( ordered ) ) {
44
+ debug ( `Fixing entities in directory ${ dir } ` ) ;
45
+ const entitiesPerDirectory = ordered [ dir ] . map ( ( e ) => e . entity ) ;
46
+ const { failed, succeeded, skipped, fixedFiles } = await fixAll (
47
+ entitiesPerDirectory ,
48
+ options ,
49
+ fixedFilesCache ,
50
+ ) ;
51
+ fixedFilesCache . push ( ...fixedFiles ) ;
52
+ handlerResult . succeeded . push ( ...succeeded ) ;
53
+ handlerResult . failed . push ( ...failed ) ;
54
+ handlerResult . skipped . push ( ...skipped ) ;
39
55
}
40
56
return handlerResult ;
41
57
}
42
58
43
- // TODO: optionally verify the deps install
44
- export async function fixIndividualRequirementsTxt (
59
+ export function getRequiredData (
45
60
entity : EntityToFix ,
46
- options : FixOptions ,
47
- ) : Promise < WithFixChangesApplied < EntityToFix > > {
48
- const fileName = entity . scanResult . identity . targetFile ;
49
- const remediationData = entity . testResult . remediation ;
50
- if ( ! remediationData ) {
61
+ ) : {
62
+ remediation : RemediationChanges ;
63
+ targetFile : string ;
64
+ workspace : Workspace ;
65
+ } {
66
+ const { remediation } = entity . testResult ;
67
+ if ( ! remediation ) {
51
68
throw new MissingRemediationDataError ( ) ;
52
69
}
53
- if ( ! fileName ) {
70
+ const { targetFile } = entity . scanResult . identity ;
71
+ if ( ! targetFile ) {
54
72
throw new MissingFileNameError ( ) ;
55
73
}
56
- const requirementsTxt = await entity . workspace . readFile ( fileName ) ;
57
- const requirementsData = parseRequirementsFile ( requirementsTxt ) ;
74
+ const { workspace } = entity ;
75
+ if ( ! workspace ) {
76
+ throw new NoFixesCouldBeAppliedError ( ) ;
77
+ }
78
+ return { targetFile, remediation, workspace } ;
79
+ }
58
80
59
- // TODO: allow handlers per fix type (later also strategies or combine with strategies)
60
- const { updatedManifest, changes } = updateDependencies (
61
- requirementsData ,
62
- remediationData . pin ,
81
+ async function fixAll (
82
+ entities : EntityToFix [ ] ,
83
+ options : FixOptions ,
84
+ fixedCache : string [ ] ,
85
+ ) : Promise < PluginFixResponse & { fixedFiles : string [ ] } > {
86
+ const handlerResult : PluginFixResponse = {
87
+ succeeded : [ ] ,
88
+ failed : [ ] ,
89
+ skipped : [ ] ,
90
+ } ;
91
+ for ( const entity of entities ) {
92
+ const targetFile = entity . scanResult . identity . targetFile ! ;
93
+ try {
94
+ const { dir, base } = pathLib . parse ( targetFile ) ;
95
+ // parse & join again to support correct separator
96
+ if ( fixedCache . includes ( pathLib . join ( dir , base ) ) ) {
97
+ handlerResult . succeeded . push ( {
98
+ original : entity ,
99
+ changes : [ { success : true , userMessage : 'Previously fixed' } ] ,
100
+ } ) ;
101
+ continue ;
102
+ }
103
+ const { changes, fixedFiles } = await applyAllFixes ( entity , options ) ;
104
+ if ( ! changes . length ) {
105
+ debug ( 'Manifest has not changed!' ) ;
106
+ throw new NoFixesCouldBeAppliedError ( ) ;
107
+ }
108
+ fixedCache . push ( ...fixedFiles ) ;
109
+ handlerResult . succeeded . push ( { original : entity , changes } ) ;
110
+ } catch ( e ) {
111
+ debug ( `Failed to fix ${ targetFile } .\nERROR: ${ e } ` ) ;
112
+ handlerResult . failed . push ( { original : entity , error : e } ) ;
113
+ }
114
+ }
115
+ return { ...handlerResult , fixedFiles : [ ] } ;
116
+ }
117
+ // TODO: optionally verify the deps install
118
+ export async function fixIndividualRequirementsTxt (
119
+ workspace : Workspace ,
120
+ dir : string ,
121
+ entryFileName : string ,
122
+ fileName : string ,
123
+ remediation : RemediationChanges ,
124
+ parsedRequirements : ParsedRequirements ,
125
+ options : FixOptions ,
126
+ directUpgradesOnly : boolean ,
127
+ ) : Promise < { changes : FixChangesSummary [ ] ; appliedRemediation : string [ ] } > {
128
+ const fullFilePath = pathLib . join ( dir , fileName ) ;
129
+ const { updatedManifest, changes, appliedRemediation } = updateDependencies (
130
+ parsedRequirements ,
131
+ remediation . pin ,
132
+ directUpgradesOnly ,
133
+ pathLib . join ( dir , entryFileName ) !== fullFilePath ? fileName : undefined ,
63
134
) ;
64
135
65
- // TODO: do this with the changes now that we only return new
66
- if ( updatedManifest === requirementsTxt ) {
67
- debug ( 'Manifest has not changed!' ) ;
68
- throw new NoFixesCouldBeAppliedError ( ) ;
136
+ if ( ! changes . length ) {
137
+ return { changes, appliedRemediation } ;
69
138
}
139
+
70
140
if ( ! options . dryRun ) {
71
141
debug ( 'Writing changes to file' ) ;
72
- await entity . workspace . writeFile ( fileName , updatedManifest ) ;
142
+ await workspace . writeFile ( pathLib . join ( dir , fileName ) , updatedManifest ) ;
73
143
} else {
74
144
debug ( 'Skipping writing changes to file in --dry-run mode' ) ;
75
145
}
76
146
77
- return {
78
- original : entity ,
79
- changes,
147
+ return { changes, appliedRemediation } ;
148
+ }
149
+
150
+ export async function applyAllFixes (
151
+ entity : EntityToFix ,
152
+ options : FixOptions ,
153
+ ) : Promise < { changes : FixChangesSummary [ ] ; fixedFiles : string [ ] } > {
154
+ const { remediation, targetFile : entryFileName , workspace } = getRequiredData (
155
+ entity ,
156
+ ) ;
157
+ const fixedFiles : string [ ] = [ ] ;
158
+ const { dir, base } = pathLib . parse ( entryFileName ) ;
159
+ const provenance = await extractProvenance ( workspace , dir , base ) ;
160
+ const upgradeChanges : FixChangesSummary [ ] = [ ] ;
161
+ const appliedUpgradeRemediation : string [ ] = [ ] ;
162
+ /* Apply all upgrades first across all files that are included */
163
+ for ( const fileName of Object . keys ( provenance ) ) {
164
+ const skipApplyingPins = true ;
165
+ const { changes, appliedRemediation } = await fixIndividualRequirementsTxt (
166
+ workspace ,
167
+ dir ,
168
+ base ,
169
+ fileName ,
170
+ remediation ,
171
+ provenance [ fileName ] ,
172
+ options ,
173
+ skipApplyingPins ,
174
+ ) ;
175
+ appliedUpgradeRemediation . push ( ...appliedRemediation ) ;
176
+ upgradeChanges . push ( ...changes ) ;
177
+ fixedFiles . push ( pathLib . join ( dir , fileName ) ) ;
178
+ }
179
+
180
+ /* Apply all left over remediation as pins in the entry targetFile */
181
+ const requirementsTxt = await workspace . readFile ( entryFileName ) ;
182
+
183
+ const toPin : RemediationChanges = filterOutAppliedUpgrades (
184
+ remediation ,
185
+ appliedUpgradeRemediation ,
186
+ ) ;
187
+ const directUpgradesOnly = false ;
188
+ const { changes : pinnedChanges } = await fixIndividualRequirementsTxt (
189
+ workspace ,
190
+ dir ,
191
+ base ,
192
+ base ,
193
+ toPin ,
194
+ parseRequirementsFile ( requirementsTxt ) ,
195
+ options ,
196
+ directUpgradesOnly ,
197
+ ) ;
198
+
199
+ return { changes : [ ...upgradeChanges , ...pinnedChanges ] , fixedFiles } ;
200
+ }
201
+
202
+ function filterOutAppliedUpgrades (
203
+ remediation : RemediationChanges ,
204
+ appliedRemediation : string [ ] ,
205
+ ) : RemediationChanges {
206
+ const pinRemediation : RemediationChanges = {
207
+ ...remediation ,
208
+ pin : { } , // delete the pin remediation so we can collect un-applied remediation
80
209
} ;
210
+ const pins = remediation . pin ;
211
+ const lowerCasedAppliedRemediation = appliedRemediation . map ( ( i ) =>
212
+ i . toLowerCase ( ) ,
213
+ ) ;
214
+ for ( const pkgAtVersion of Object . keys ( pins ) ) {
215
+ if ( ! lowerCasedAppliedRemediation . includes ( pkgAtVersion . toLowerCase ( ) ) ) {
216
+ pinRemediation . pin [ pkgAtVersion ] = pins [ pkgAtVersion ] ;
217
+ }
218
+ }
219
+ return pinRemediation ;
220
+ }
221
+
222
+ function sortByDirectory (
223
+ entities : EntityToFix [ ] ,
224
+ ) : {
225
+ [ dir : string ] : Array < {
226
+ entity : EntityToFix ;
227
+ dir : string ;
228
+ base : string ;
229
+ ext : string ;
230
+ root : string ;
231
+ name : string ;
232
+ } > ;
233
+ } {
234
+ const mapped = entities . map ( ( e ) => ( {
235
+ entity : e ,
236
+ ...pathLib . parse ( e . scanResult . identity . targetFile ! ) ,
237
+ } ) ) ;
238
+
239
+ const sorted = sortBy ( mapped , 'dir' ) ;
240
+ return groupBy ( sorted , 'dir' ) ;
81
241
}
0 commit comments