Skip to content

Commit 2156be9

Browse files
Merge pull request #916 from ibi-group/mobility-data-validator
Add MobilityData Validator
2 parents 1dde974 + 8ec815b commit 2156be9

25 files changed

+318
-25
lines changed

__tests__/e2e/server/Dockerfile

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# syntax=docker/dockerfile:1
2-
FROM maven:3.8.6-openjdk-8
2+
FROM maven:3.8.7-openjdk-18
3+
34
WORKDIR /datatools
45

56
ARG E2E_AUTH0_USERNAME
@@ -32,6 +33,7 @@ ARG AWS_SECRET_ACCESS_KEY
3233

3334
# Grab latest dev build of Datatools Server
3435
RUN git clone https://github.com/ibi-group/datatools-server.git
36+
RUN microdnf install wget
3537
WORKDIR /datatools/datatools-server
3638

3739
RUN mvn package -DskipTests

__tests__/test-utils/mock-data/manager.js

+1
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,7 @@ export const mockFeedVersion = {
340340
feedVersionId: 'mock-feed-version-id',
341341
loadFailureReason: null,
342342
loadStatus: 'SUCCESS',
343+
mobilityDataResult: {},
343344
routeCount: 10,
344345
startDate: '20180801',
345346
stopCount: 237,

lib/admin/components/OrganizationSettings.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import moment from 'moment'
88

99
import * as organizationActions from '../actions/organizations'
1010
import {getComponentMessages} from '../../common/util/config'
11-
import toSentenceCase from '../../common/util/to-sentence-case'
11+
import toSentenceCase from '../../common/util/text'
1212
import type {Organization, Project} from '../../types'
1313

1414
type Props = {

lib/alerts/components/AlertEditor.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import ManagerPage from '../../common/components/ManagerPage'
1515
import PageNotFound from '../../common/components/PageNotFound'
1616
import {isModuleEnabled} from '../../common/util/config'
1717
import {checkEntitiesForFeeds} from '../../common/util/permissions'
18-
import toSentenceCase from '../../common/util/to-sentence-case'
18+
import toSentenceCase from '../../common/util/text'
1919
import GtfsMapSearch from '../../gtfs/components/gtfsmapsearch'
2020
import GlobalGtfsFilter from '../../gtfs/containers/GlobalGtfsFilter'
2121
import {CAUSES, EFFECTS, isNew} from '../util'

lib/alerts/components/AlertsList.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ import AlertPreview from './AlertPreview'
1818
import Loading from '../../common/components/Loading'
1919
import OptionButton from '../../common/components/OptionButton'
2020
import { getFeedId } from '../../common/util/modules'
21-
import toSentenceCase from '../../common/util/to-sentence-case'
21+
import toSentenceCase from '../../common/util/text'
2222
import { FILTERS, SORT_OPTIONS } from '../util'
2323

2424
import type {Props as ContainerProps} from '../containers/VisibleAlertsList'

lib/common/util/text.js

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// @flow
2+
3+
import toLower from 'lodash/toLower'
4+
import upperFirst from 'lodash/upperFirst'
5+
6+
export default function toSentenceCase (s: string): string {
7+
return upperFirst(toLower(s))
8+
}
9+
10+
/**
11+
* This method takes a string like expires_in_7days and ensures
12+
* that 7days is replaced with 7 days
13+
*/
14+
// $FlowFixMe flow needs to learn about new es2021 features!
15+
export function spaceOutNumbers (s: string): string {
16+
return s.replaceAll('_', ' ')
17+
.split(/(?=[1-9])/)
18+
.join(' ')
19+
.toLowerCase()
20+
}

lib/common/util/to-sentence-case.js

-8
This file was deleted.

lib/editor/components/EditorInput.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ import {doesNotExist} from '../util/validation'
1515
import TimezoneSelect from '../../common/components/TimezoneSelect'
1616
import LanguageSelect from '../../common/components/LanguageSelect'
1717
import {getComponentMessages} from '../../common/util/config'
18-
import toSentenceCase from '../../common/util/to-sentence-case'
18+
import toSentenceCase from '../../common/util/text'
1919
import type {Entity, Feed, GtfsSpecField, GtfsAgency, GtfsStop} from '../../types'
2020
import type {EditorTables} from '../../types/reducers'
2121

lib/editor/components/ScheduleExceptionForm.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import Select from 'react-select'
1313
import FlipMove from 'react-flip-move'
1414

1515
import {updateActiveGtfsEntity} from '../actions/active'
16-
import toSentenceCase from '../../common/util/to-sentence-case'
16+
import toSentenceCase from '../../common/util/text'
1717
import {getRangesForDates} from '../../common/util/exceptions'
1818
import {EXCEPTION_EXEMPLARS} from '../util'
1919
import {getTableById} from '../util/gtfs'

lib/editor/components/pattern/EditSettings.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import Rcslider from 'rc-slider'
66

77
import {updateEditSetting} from '../../actions/active'
88
import {CLICK_OPTIONS} from '../../util'
9-
import toSentenceCase from '../../../common/util/to-sentence-case'
9+
import toSentenceCase from '../../../common/util/text'
1010
import type {EditSettingsState} from '../../../types/reducers'
1111

1212
type Props = {

lib/manager/components/FeedSourceSettings.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import {LinkContainer} from 'react-router-bootstrap'
1212

1313
import * as feedsActions from '../actions/feeds'
1414
import {getComponentMessages, isExtensionEnabled} from '../../common/util/config'
15-
import toSentenceCase from '../../common/util/to-sentence-case'
15+
import toSentenceCase from '../../common/util/text'
1616
import type {Feed, Project} from '../../types'
1717
import type {ManagerUserState} from '../../types/reducers'
1818

lib/manager/components/validation/GtfsValidationViewer.js

+15-2
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ import {
1717
import Loading from '../../../common/components/Loading'
1818
import OptionButton from '../../../common/components/OptionButton'
1919
import {getComponentMessages} from '../../../common/util/config'
20-
import toSentenceCase from '../../../common/util/to-sentence-case'
20+
import toSentenceCase from '../../../common/util/text'
2121
import {
2222
BLOCKING_ERROR_TYPES,
2323
getTableFatalExceptions,
@@ -30,6 +30,7 @@ import type {Props as FeedVersionViewerProps} from '../version/FeedVersionViewer
3030
import type {ValidationResult} from '../../../types'
3131

3232
import ValidationErrorItem from './ValidationErrorItem'
33+
import MobilityDataValidationResult from './MobilityDataValidationResult'
3334

3435
const DEFAULT_LIMIT = 10
3536

@@ -266,9 +267,21 @@ export default class GtfsValidationViewer extends Component<Props, State> {
266267
</Alert>
267268
}
268269
<Panel>
269-
<Panel.Heading><Panel.Title componentClass='h2'>{this.messages('title')}</Panel.Title></Panel.Heading>
270+
<Panel.Heading>
271+
<Panel.Title componentClass='h2'>{this.messages('title')}</Panel.Title>
272+
</Panel.Heading>
270273
{validationContent}
271274
</Panel>
275+
<Panel><Panel.Heading>Mobility Data Validation Issues</Panel.Heading>
276+
<ListGroup>
277+
{version.mobilityDataResult && version.mobilityDataResult.notices.map(notice => (
278+
<MobilityDataValidationResult notice={notice} />
279+
))}
280+
{(!version.mobilityDataResult || version.mobilityDataResult.notices.length === 0) && <ListGroupItem className='validation-item'>
281+
The MobilityData validator has not produced any errors or warnings. This may be because the validator is still running.
282+
</ListGroupItem>}
283+
</ListGroup>
284+
</Panel>
272285
</div>
273286
)
274287
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
/* eslint-disable jsx-a11y/no-noninteractive-element-interactions */
2+
/* eslint-disable no-fallthrough */
3+
import React, { useState } from 'react'
4+
import Icon from '@conveyal/woonerf/components/icon'
5+
import { ListGroupItem, Table } from 'react-bootstrap'
6+
import Markdown from 'markdown-to-jsx'
7+
8+
import toSentenceCase, { spaceOutNumbers } from '../../../common/util/text'
9+
import {
10+
mobilityDataValidationErrorMapping,
11+
validationErrorIconLookup
12+
} from '../../util/version'
13+
14+
import rules from './rules.json'
15+
16+
// from https://stackoverflow.com/a/4149671
17+
function unCamelCase (s) {
18+
return s
19+
.split(/(?=[A-Z])/)
20+
.join(' ')
21+
.toLowerCase()
22+
}
23+
24+
const NoticeTable = ({ headerOverides = {
25+
'stopSequence1': 'Stop seq-uence 1',
26+
'stopSequence2': 'Stop seq-uence 2'
27+
}, notices }) => {
28+
if (notices.length === 0) return null
29+
30+
const headers = Object.keys(notices[0])
31+
32+
return (
33+
<Table bordered className='table-fixed' fill hover striped style={{display: 'relative', borderCollapse: 'collapse'}}>
34+
<thead>
35+
<tr>
36+
{headers.map((header) => (
37+
<th style={{position: 'sticky', top: 0, background: '#fefefe'}}>
38+
{headerOverides[header] || toSentenceCase(unCamelCase(header))}
39+
</th>
40+
))}
41+
</tr>
42+
</thead>
43+
<tbody>
44+
{notices.map((notice) => (
45+
<tr>
46+
{headers.map((header, index) => {
47+
const FieldWrapper =
48+
(header === 'fieldValue' || header === 'message') ? 'pre' : React.Fragment
49+
50+
let field = notice[header]
51+
if (header.endsWith('Km') || header.endsWith('Kph')) {
52+
field = Math.round(field)
53+
}
54+
55+
return (
56+
<td key={`${header}-${index}`}>
57+
<FieldWrapper>{field}</FieldWrapper>
58+
</td>
59+
)
60+
})}
61+
</tr>
62+
))}
63+
</tbody>
64+
</Table>
65+
)
66+
}
67+
68+
// eslint-disable-next-line complexity
69+
const renderNoticeDetail = (notice) => {
70+
switch (notice.code) {
71+
case 'too_many_rows':
72+
notice.csvRowNumber = notice.rowNumber
73+
case 'fare_transfer_rule_duration_limit_type_without_duration_limit':
74+
case 'fare_transfer_rule_duration_limit_without_type':
75+
case 'fare_transfer_rule_missing_transfer_count':
76+
case 'fare_transfer_rule_with_forbidden_transfer_count':
77+
notice.filename = 'fare_transfer_rules.txt'
78+
case 'empty_file':
79+
case 'emtpy_row':
80+
case 'missing_timepoint_column':
81+
case 'missing_required_file':
82+
case 'missing_recommended_file':
83+
case 'unknown_file':
84+
return (
85+
<ul>
86+
{notice.sampleNotices.map((notice) => (
87+
<li>
88+
{notice.filename}
89+
{notice.csvRowNumber && `: row ${notice.csvRowNumber}`}
90+
</li>
91+
))}
92+
</ul>
93+
)
94+
default:
95+
return (
96+
<NoticeTable notices={notice.sampleNotices} />
97+
)
98+
}
99+
}
100+
101+
const MobilityDataValidationResult = ({notice}) => {
102+
const rule = rules.find((rd) => rd.rule === notice.code)
103+
if (!rule) return null
104+
105+
const errorClass = `gtfs-error-${mobilityDataValidationErrorMapping[notice.severity]}`
106+
const [expanded, setExpanded] = useState(notice.totalNotices < 2)
107+
108+
const onRowSelect = () => setExpanded(!expanded)
109+
110+
return (
111+
<ListGroupItem style={{ padding: 0 }}>
112+
<div style={{ padding: '10px 15px' }}>
113+
<h4
114+
onClick={onRowSelect}
115+
onKeyDown={onRowSelect}
116+
style={{ cursor: 'pointer' }}
117+
>
118+
<span
119+
className={`buffer-icon ${errorClass}`}
120+
title={`${toSentenceCase(notice.severity)} priority`}
121+
>
122+
<Icon
123+
type={validationErrorIconLookup[mobilityDataValidationErrorMapping[notice.severity]]}
124+
/>
125+
</span>
126+
{toSentenceCase(spaceOutNumbers(notice.code))}
127+
<span className={errorClass}>
128+
{' '}
129+
&mdash; {notice.totalNotices} case
130+
{notice.totalNotices > 1 ? 's' : ''} found
131+
</span>
132+
<span className={`pull-right`}>
133+
<Icon type={expanded ? 'caret-up' : 'caret-down'} />
134+
</span>
135+
</h4>
136+
{expanded && (
137+
<>
138+
<p><Markdown>{rule.description}</Markdown></p>
139+
<p>
140+
<a
141+
href={`https://github.com/MobilityData/gtfs-validator/blob/master/RULES.md#${notice.code}`}
142+
target='_blank'
143+
rel='noreferrer'
144+
>
145+
More details
146+
</a>
147+
</p>
148+
</>
149+
)}
150+
</div>
151+
<div>{expanded && renderNoticeDetail(notice)}</div>
152+
</ListGroupItem>
153+
)
154+
}
155+
156+
export default MobilityDataValidationResult

lib/manager/components/validation/ServicePerModeChart.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import React, { Component } from 'react'
44
import moment from 'moment'
55

66
import Loading from '../../../common/components/Loading'
7-
import toSentenceCase from '../../../common/util/to-sentence-case'
7+
import toSentenceCase from '../../../common/util/text'
88
import {getChartMax, getChartPeriod} from '../../util'
99

1010
import type {ValidationResult} from '../../../types'

lib/manager/components/validation/TripsChart.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import React, { Component } from 'react'
44
import moment from 'moment'
55

66
import Loading from '../../../common/components/Loading'
7-
import toSentenceCase from '../../../common/util/to-sentence-case'
7+
import toSentenceCase from '../../../common/util/text'
88
import {getChartMax, getChartPeriod} from '../../util'
99

1010
import type {ValidationResult} from '../../../types'

lib/manager/components/validation/rules.json

+1
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)