Skip to content

Commit 4b4a12a

Browse files
mkasprzykmkasprzyk
committed
Add Flake8 support for imported reports
2 parents 40f4331 + 97bced7 commit 4b4a12a

File tree

14 files changed

+1418
-2
lines changed

14 files changed

+1418
-2
lines changed

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/PythonPlugin.java

+23-1
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,17 @@
2323
import org.sonar.api.PropertyType;
2424
import org.sonar.api.SonarProduct;
2525
import org.sonar.api.SonarRuntime;
26+
27+
import com.google.common.collect.ImmutableList;
28+
2629
import org.sonar.api.config.PropertyDefinition;
2730
import org.sonar.api.resources.Qualifiers;
2831
import org.sonar.api.utils.Version;
2932
import org.sonar.plugins.python.bandit.BanditRulesDefinition;
3033
import org.sonar.plugins.python.bandit.BanditSensor;
3134
import org.sonar.plugins.python.coverage.PythonCoverageSensor;
35+
import org.sonar.plugins.python.flake8.Flake8RuleRepository;
36+
import org.sonar.plugins.python.flake8.Flake8ImportSensor;
3237
import org.sonar.plugins.python.pylint.PylintConfiguration;
3338
import org.sonar.plugins.python.pylint.PylintImportSensor;
3439
import org.sonar.plugins.python.pylint.PylintRuleRepository;
@@ -45,6 +50,7 @@ public class PythonPlugin implements Plugin {
4550
private static final String TEST_AND_COVERAGE = "Tests and Coverage";
4651
private static final String EXTERNAL_ANALYZERS_CATEGORY = "External Analyzers";
4752
private static final String PYLINT = "Pylint";
53+
private static final String FLAKE8 = "Flake8";
4854
private static final String DEPRECATED_PREFIX = "DEPRECATED : Use " + PythonCoverageSensor.REPORT_PATHS_KEY + " instead. ";
4955

5056
public static final String FILE_SUFFIXES_KEY = "sonar.python.file.suffixes";
@@ -79,6 +85,7 @@ public void define(Context context) {
7985
addXUnitExtensions(context);
8086
addPylintExtensions(context);
8187
addBanditExtensions(context);
88+
addFlake8Extensions(context);
8289
}
8390
}
8491

@@ -185,4 +192,19 @@ private static void addBanditExtensions(Context context) {
185192
}
186193
}
187194

188-
}
195+
private static void addFlake8Extensions(Context context) {
196+
context.addExtensions(
197+
PropertyDefinition.builder(Flake8ImportSensor.REPORT_PATH_KEY)
198+
.index(42)
199+
.name("Flake8 reports")
200+
.description("Path to Flake8 report file, relative to projects root")
201+
.category(PYTHON_CATEGORY)
202+
.subCategory(FLAKE8)
203+
.onQualifiers(Qualifiers.PROJECT)
204+
.defaultValue("")
205+
.build(),
206+
Flake8ImportSensor.class,
207+
Flake8RuleRepository.class);
208+
}
209+
210+
}

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/flake8/Flake8ImportSensor.java

+147
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2020 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.plugins.python.flake8;
21+
22+
import java.io.File;
23+
import java.io.IOException;
24+
import java.util.HashSet;
25+
import java.util.LinkedList;
26+
import java.util.List;
27+
import java.util.Scanner;
28+
import java.util.Set;
29+
import javax.annotation.Nullable;
30+
import org.sonar.api.batch.fs.FileSystem;
31+
import org.sonar.api.batch.fs.InputFile;
32+
import org.sonar.api.batch.rule.ActiveRule;
33+
import org.sonar.api.batch.sensor.SensorContext;
34+
import org.sonar.api.batch.sensor.SensorDescriptor;
35+
import org.sonar.api.batch.sensor.issue.NewIssue;
36+
import org.sonar.api.config.Configuration;
37+
import org.sonar.api.rule.RuleKey;
38+
import org.sonar.api.utils.log.Logger;
39+
import org.sonar.api.utils.log.Loggers;
40+
import org.sonar.plugins.python.PythonReportSensor;
41+
import org.sonar.plugins.python.warnings.AnalysisWarningsWrapper;
42+
43+
public class Flake8ImportSensor extends PythonReportSensor {
44+
public static final String REPORT_PATH_KEY = "sonar.python.flake8.reportPath";
45+
private static final String DEFAULT_REPORT_PATH = "flake8-reports/flake8-result-*.txt";
46+
47+
private static final Logger LOG = Loggers.get(Flake8ImportSensor.class);
48+
private static final Flake8RuleParser flake8Rules = new Flake8RuleParser(Flake8RuleRepository.RULES_FILE);
49+
private static final Set<String> warningAlreadyLogged = new HashSet<>();
50+
51+
public Flake8ImportSensor(Configuration conf, AnalysisWarningsWrapper analysisWarnings) {
52+
super(conf, analysisWarnings, "Flake8");
53+
}
54+
55+
@Override
56+
public void describe(SensorDescriptor descriptor) {
57+
super.describe(descriptor);
58+
descriptor
59+
.createIssuesForRuleRepository(Flake8RuleRepository.REPOSITORY_KEY)
60+
.onlyWhenConfiguration(conf -> conf.hasKey(REPORT_PATH_KEY));
61+
}
62+
63+
@Override
64+
protected String reportPathKey() {
65+
return REPORT_PATH_KEY;
66+
}
67+
68+
@Override
69+
protected String defaultReportPath() {
70+
return DEFAULT_REPORT_PATH;
71+
}
72+
73+
@Override
74+
protected void processReports(final SensorContext context, List<File> reports) {
75+
List<Issue> issues = new LinkedList<>();
76+
for (File report : reports) {
77+
try {
78+
issues.addAll(parse(report, context.fileSystem()));
79+
} catch (java.io.FileNotFoundException e) {
80+
LOG.error("Report '{}' cannot be found, details: '{}'", report, e);
81+
} catch (IOException e) {
82+
LOG.error("Report '{}' cannot be read, details: '{}'", report, e);
83+
}
84+
}
85+
86+
saveIssues(issues, context);
87+
}
88+
89+
private static List<Issue> parse(File report, FileSystem fileSystem) throws IOException {
90+
List<Issue> issues = new LinkedList<>();
91+
92+
Flake8ReportParser parser = new Flake8ReportParser();
93+
Scanner sc;
94+
for (sc = new Scanner(report.toPath(), fileSystem.encoding().name()); sc.hasNext(); ) {
95+
String line = sc.nextLine();
96+
Issue issue = parser.parseLine(line);
97+
if (issue != null) {
98+
issues.add(issue);
99+
}
100+
}
101+
sc.close();
102+
return issues;
103+
}
104+
105+
private static void saveIssues(List<Issue> issues, SensorContext context) {
106+
FileSystem fileSystem = context.fileSystem();
107+
for (Issue flake8Issue : issues) {
108+
String filepath = flake8Issue.getFilename();
109+
InputFile pyfile = fileSystem.inputFile(fileSystem.predicates().hasPath(filepath));
110+
if (pyfile != null) {
111+
ActiveRule rule = context.activeRules().find(RuleKey.of(Flake8RuleRepository.REPOSITORY_KEY, flake8Issue.getRuleId()));
112+
processRule(flake8Issue, pyfile, rule, context);
113+
} else {
114+
LOG.warn("Cannot find the file '{}' in SonarQube, ignoring violation", filepath);
115+
}
116+
}
117+
}
118+
119+
public static void processRule(Issue flake8Issue, InputFile pyfile, @Nullable ActiveRule rule, SensorContext context) {
120+
if (rule != null) {
121+
NewIssue newIssue = context
122+
.newIssue()
123+
.forRule(rule.ruleKey());
124+
newIssue.at(
125+
newIssue.newLocation()
126+
.on(pyfile)
127+
.at(pyfile.selectLine(flake8Issue.getLine()))
128+
.message(flake8Issue.getDescription()));
129+
newIssue.save();
130+
} else if (!flake8Rules.hasRuleDefinition(flake8Issue.getRuleId())) {
131+
logUnknownRuleWarning(flake8Issue.getRuleId());
132+
}
133+
}
134+
135+
private static void logUnknownRuleWarning(String ruleId) {
136+
if (!warningAlreadyLogged.contains(ruleId)) {
137+
warningAlreadyLogged.add(ruleId);
138+
LOG.warn("Flake8 rule '{}' is unknown in Sonar", ruleId);
139+
}
140+
}
141+
142+
// Visible for testing
143+
static void clearLoggedWarnings() {
144+
warningAlreadyLogged.clear();
145+
}
146+
147+
}

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/flake8/Flake8ReportParser.java

+61
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2020 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.plugins.python.flake8;
21+
22+
import java.util.regex.Matcher;
23+
import java.util.regex.Pattern;
24+
import org.sonar.api.utils.log.Logger;
25+
import org.sonar.api.utils.log.Loggers;
26+
27+
public class Flake8ReportParser {
28+
private static final Pattern PATTERN = Pattern.compile("([^:]+):([0-9]+):([0-9]+): (\\S+) (.*)");
29+
private static final Logger LOG = Loggers.get(Flake8ReportParser.class);
30+
31+
public Issue parseLine(String line) {
32+
// Parse the output of Flake8. Example of the format:
33+
//
34+
// app/start.py:42:45: Q000 Remove bad quotes
35+
// ...
36+
37+
Issue issue = null;
38+
39+
int linenr;
40+
String filename = null;
41+
String ruleid = null;
42+
String descr = null;
43+
44+
if (line.length() > 0) {
45+
Matcher m = PATTERN.matcher(line);
46+
if (m.matches() && m.groupCount() == 5) {
47+
filename = m.group(1);
48+
linenr = Integer.valueOf(m.group(2));
49+
ruleid = m.group(4);
50+
descr = m.group(5);
51+
issue = new Issue(filename, linenr, ruleid, descr);
52+
} else {
53+
LOG.debug("Cannot parse the line: {}", line);
54+
}
55+
} else {
56+
LOG.trace("Classifying as detail and ignoring line '{}'", line);
57+
}
58+
return issue;
59+
}
60+
61+
}

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/flake8/Flake8RuleParser.java

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2020 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.plugins.python.flake8;
21+
22+
import java.io.IOException;
23+
import java.io.InputStream;
24+
import java.util.HashSet;
25+
import java.util.Set;
26+
import javax.xml.stream.XMLEventReader;
27+
import javax.xml.stream.XMLStreamException;
28+
import javax.xml.stream.events.StartElement;
29+
import javax.xml.stream.events.XMLEvent;
30+
import org.sonar.api.utils.log.Logger;
31+
import org.sonar.api.utils.log.Loggers;
32+
import org.sonarsource.analyzer.commons.xml.SafetyFactory;
33+
34+
public class Flake8RuleParser {
35+
36+
private static final Logger LOG = Loggers.get(Flake8RuleParser.class);
37+
private Set<String> definedRulesId = new HashSet<>();
38+
private StringBuilder currentKey = new StringBuilder();
39+
40+
public Flake8RuleParser(String rulesPath) {
41+
try (InputStream inputStream = getClass().getResourceAsStream(rulesPath)) {
42+
XMLEventReader reader = SafetyFactory.createXMLInputFactory().createXMLEventReader(inputStream);
43+
while (reader.hasNext()) {
44+
onXmlEvent(reader.nextEvent());
45+
}
46+
} catch (IOException | XMLStreamException | IllegalArgumentException e) {
47+
LOG.warn("Unable to parse the Flake8 rules definition XML file");
48+
}
49+
50+
if (definedRulesId.isEmpty()) {
51+
LOG.warn("No rule key found for Flake8");
52+
}
53+
}
54+
55+
private void onXmlEvent(XMLEvent event) {
56+
if (event.isStartElement()) {
57+
StartElement element = event.asStartElement();
58+
String elementName = element.getName().getLocalPart();
59+
if ("key".equals(elementName)) {
60+
currentKey = new StringBuilder();
61+
}
62+
} else if (event.isCharacters()) {
63+
currentKey.append(event.asCharacters().getData());
64+
} else if (event.isEndElement() && "key".equals(event.asEndElement().getName().getLocalPart())) {
65+
definedRulesId.add(currentKey.toString());
66+
}
67+
}
68+
69+
public boolean hasRuleDefinition(String ruleId) {
70+
return definedRulesId.contains(ruleId);
71+
}
72+
73+
}

‎sonar-python-plugin/src/main/java/org/sonar/plugins/python/flake8/Flake8RuleRepository.java

+53
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* SonarQube Python Plugin
3+
* Copyright (C) 2011-2020 SonarSource SA
4+
* mailto:info AT sonarsource DOT com
5+
*
6+
* This program is free software; you can redistribute it and/or
7+
* modify it under the terms of the GNU Lesser General Public
8+
* License as published by the Free Software Foundation; either
9+
* version 3 of the License, or (at your option) any later version.
10+
*
11+
* This program is distributed in the hope that it will be useful,
12+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
13+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14+
* Lesser General Public License for more details.
15+
*
16+
* You should have received a copy of the GNU Lesser General Public License
17+
* along with this program; if not, write to the Free Software Foundation,
18+
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
19+
*/
20+
package org.sonar.plugins.python.flake8;
21+
22+
import java.util.HashMap;
23+
import java.util.Map;
24+
import java.util.Scanner;
25+
import org.sonar.api.server.rule.RulesDefinition;
26+
import org.sonar.api.server.rule.RulesDefinitionXmlLoader;
27+
import org.sonar.plugins.python.Python;
28+
29+
import static java.nio.charset.StandardCharsets.UTF_8;
30+
31+
public class Flake8RuleRepository implements RulesDefinition {
32+
33+
public static final String REPOSITORY_NAME = "Flake8";
34+
public static final String REPOSITORY_KEY = REPOSITORY_NAME;
35+
36+
public static final String RULES_FILE = "/org/sonar/plugins/python/flake8/rules.xml";
37+
38+
private final RulesDefinitionXmlLoader xmlLoader;
39+
40+
public Flake8RuleRepository(RulesDefinitionXmlLoader xmlLoader) {
41+
this.xmlLoader = xmlLoader;
42+
}
43+
44+
@Override
45+
public void define(Context context) {
46+
NewRepository repository = context
47+
.createRepository(REPOSITORY_KEY, Python.KEY)
48+
.setName(REPOSITORY_NAME);
49+
xmlLoader.load(repository, getClass().getResourceAsStream(RULES_FILE), UTF_8.name());
50+
repository.done();
51+
}
52+
53+
}

0 commit comments

Comments
 (0)