Skip to content

Commit 0fce04a

Browse files
RelloRello
committed
Datasource: Excel (xls, xlsx, ods) #115 
1 parent f45de30 commit 0fce04a

File tree

748 files changed

+128055
-34
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

748 files changed

+128055
-34
lines changed

‎CHANGELOG.md

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,15 @@
11
# Changelog
22

33
## 3.3.3 - 2021-02-24
4-
54
### Fixed
6-
75
- Database issue on NC21 installation [#113](https://github.com/rello/analytics/issues/113)
86

97
## 3.3.2 - 2021-02-17
10-
118
### Fixed
12-
139
- Dataload issue with "External file" [#110](https://github.com/rello/analytics/issues/110)
1410

1511
## 3.3.0 - 2021-02-13
16-
1712
### Added
18-
1913
- First Start Wizzard [#103](https://github.com/rello/analytics/issues/103)
2014
- Export / import reports (incl data) [#100](https://github.com/rello/analytics/issues/100)
2115
- Parameter to skip header rows in csv and file datasource [#97](https://github.com/rello/analytics/issues/97)

‎composer.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
},
77
"require-dev": {
88
"christophwurst/nextcloud": "^20.0.7",
9-
"symfony/symfony": "^4.4.0",
9+
"symfony/symfony": "^4.4.19",
1010
"guzzlehttp/guzzle": "^6.3.0"
1111
}
1212
}

‎js/advanced.js

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ OCA.Analytics.Advanced.Dataload = {
128128
document.getElementById(fieldValue) ? document.getElementById(fieldValue).value = fieldValues[fieldValue] : null;
129129
}
130130

131-
if (dataload['datasource'] === OCA.Analytics.TYPE_INTERNAL_FILE) {
131+
if (dataload['datasource'] === OCA.Analytics.TYPE_INTERNAL_FILE || dataload['datasource'] === OCA.Analytics.TYPE_EXCEL) {
132132
document.getElementById('link').addEventListener('click', OCA.Analytics.Sidebar.Dataset.handleFilepicker);
133133
}
134134

@@ -223,7 +223,6 @@ OCA.Analytics.Advanced.Dataload = {
223223
'info',
224224
OC.dialogs.OK_BUTTON,
225225
function () {
226-
227226
},
228227
true,
229228
true
@@ -241,14 +240,19 @@ OCA.Analytics.Advanced.Dataload = {
241240
},
242241
success: function (data) {
243242
if (mode === 'simulate') {
244-
document.querySelector("[id*=oc-dialog-]").innerHTML = JSON.stringify(data.data);
243+
if (parseInt(data.error) === 0) {
244+
document.querySelector("[id*=oc-dialog-]").innerHTML = JSON.stringify(data.data);
245+
} else {
246+
document.querySelector("[id*=oc-dialog-]").innerHTML = 'Error: ' + data.error;
247+
}
245248
} else {
246-
if (data.error === 0) {
247-
OCA.Analytics.UI.notification('success', data.insert + t('analytics', ' records inserted, ') + data.update + t('analytics', ' records updated'));
248-
//document.querySelector('#navigationDatasets [data-id="' + datasetId + '"]').click();
249+
let messageType;
250+
if (parseInt(data.error) === 0) {
251+
messageType = 'success';
249252
} else {
250-
OCA.Analytics.UI.notification('error', data.error);
253+
messageType = 'error';
251254
}
255+
OCA.Analytics.UI.notification(messageType, data.insert + t('analytics', ' records inserted, ') + data.update + t('analytics', ' records updated, ') + data.error + t('analytics', ' errors'));
252256
}
253257
}
254258
});

‎js/app.js

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@ if (!OCA.Analytics) {
2828
TYPE_GIT: 3,
2929
TYPE_EXTERNAL_FILE: 4,
3030
TYPE_EXTERNAL_REGEX: 5,
31+
TYPE_EXCEL: 7,
3132
TYPE_SHARED: 99,
3233
SHARE_TYPE_USER: 0,
3334
SHARE_TYPE_GROUP: 1,
@@ -542,8 +543,7 @@ OCA.Analytics.Backend = {
542543
}
543544

544545
document.title = data.options.name + ' @ ' + OCA.Analytics.initialDocumentTitle;
545-
if (data.status !== 'nodata') {
546-
546+
if (data.status !== 'nodata' && parseInt(data.error) === 0) {
547547
let visualization = data.options.visualization;
548548
if (visualization === 'chart') {
549549
document.getElementById('optionsIcon').style.removeProperty('display');
@@ -557,6 +557,9 @@ OCA.Analytics.Backend = {
557557
}
558558
} else {
559559
document.getElementById('noDataContainer').style.removeProperty('display');
560+
if (parseInt(data.error) !== 0) {
561+
OCA.Analytics.UI.notification('error', data.error);
562+
}
560563
}
561564
}
562565
});

‎js/navigation.js

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ OCA.Analytics.Navigation = {
5555
let a = document.createElement('a');
5656
a.setAttribute('href', '#/r/' + data['id']);
5757
let typeINT = parseInt(data['type']);
58-
if (typeINT === OCA.Analytics.TYPE_INTERNAL_FILE) {
58+
if (typeINT === OCA.Analytics.TYPE_INTERNAL_FILE || typeINT === OCA.Analytics.TYPE_EXCEL) {
5959
typeIcon = 'icon-file';
6060
} else if (typeINT === OCA.Analytics.TYPE_INTERNAL_DB) {
6161
typeIcon = 'icon-projects';
@@ -363,6 +363,8 @@ OCA.Analytics.Navigation = {
363363
navigationItem.parentElement.parentElement.parentElement.classList.add('open');
364364
}
365365
navigationItem.click();
366+
//todo open the exit dialog for a new data set
367+
//document.querySelector('#navigationMenu[data-id="292"] #navigationMenuEdit').click()
366368
}
367369
}
368370
});

‎js/sidebar.js

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -263,20 +263,36 @@ OCA.Analytics.Sidebar.Dataset = {
263263

264264
document.getElementById('datasetDatasourceSection').appendChild(OCA.Analytics.Datasource.buildOptionsForm(type));
265265

266-
if (type === OCA.Analytics.TYPE_INTERNAL_FILE) {
266+
if (type === OCA.Analytics.TYPE_INTERNAL_FILE || type === OCA.Analytics.TYPE_EXCEL) {
267267
document.getElementById('link').addEventListener('click', OCA.Analytics.Sidebar.Dataset.handleFilepicker);
268268
}
269269
}
270270
},
271271

272272
handleFilepicker: function () {
273+
let type;
274+
if (document.getElementById('dataloadDetail') !== null) {
275+
let dataloadId = document.getElementById('dataloadDetail').dataset.id;
276+
type = OCA.Analytics.Advanced.Dataload.dataloadArray.find(x => parseInt(x.id) === parseInt(dataloadId));
277+
} else {
278+
type = parseInt(document.getElementById('app-sidebar').dataset.type);
279+
}
280+
281+
let mime;
282+
if (type === OCA.Analytics.TYPE_INTERNAL_FILE) {
283+
mime = ['text/csv', 'text/plain'];
284+
} else if (type === OCA.Analytics.TYPE_EXCEL) {
285+
mime = ['application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
286+
'application/vnd.oasis.opendocument.spreadsheet',
287+
'application/vnd.ms-excel'];
288+
}
273289
OC.dialogs.filepicker(
274290
t('analytics', 'Select file'),
275291
function (path) {
276292
document.getElementById('link').value = path;
277293
},
278294
false,
279-
['text/csv', 'text/plain'],
295+
mime,
280296
true,
281297
1);
282298
},

‎lib/Controller/DatasourceController.php

Lines changed: 20 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
namespace OCA\Analytics\Controller;
1313

1414
use OCA\Analytics\Datasource\DatasourceEvent;
15+
use OCA\Analytics\Datasource\Excel;
1516
use OCA\Analytics\Datasource\ExternalFile;
1617
use OCA\Analytics\Datasource\File;
1718
use OCA\Analytics\Datasource\Github;
@@ -32,6 +33,7 @@ class DatasourceController extends Controller
3233
private $ExternalFileService;
3334
private $RegexService;
3435
private $JsonService;
36+
private $ExcelService;
3537
/** @var IEventDispatcher */
3638
private $dispatcher;
3739
private $l10n;
@@ -43,6 +45,7 @@ class DatasourceController extends Controller
4345
const DATASET_TYPE_EXTERNAL_FILE = 4;
4446
const DATASET_TYPE_REGEX = 5;
4547
const DATASET_TYPE_JSON = 6;
48+
const DATASET_TYPE_EXCEL = 7;
4649

4750
public function __construct(
4851
string $AppName,
@@ -53,6 +56,7 @@ public function __construct(
5356
Regex $RegexService,
5457
Json $JsonService,
5558
ExternalFile $ExternalFileService,
59+
Excel $ExcelService,
5660
IL10N $l10n,
5761
IEventDispatcher $dispatcher
5862
)
@@ -64,6 +68,7 @@ public function __construct(
6468
$this->RegexService = $RegexService;
6569
$this->FileService = $FileService;
6670
$this->JsonService = $JsonService;
71+
$this->ExcelService = $ExcelService;
6772
$this->dispatcher = $dispatcher;
6873
$this->l10n = $l10n;
6974
}
@@ -116,16 +121,20 @@ public function getTemplates()
116121
*/
117122
public function read(int $datasourceId, $option)
118123
{
119-
$result = $this->getDatasources()[$datasourceId]->readData($option);
120-
121-
if (isset($option['timestamp']) and $option['timestamp'] === 'true') {
122-
// if datasource should be timestamped/snapshoted
123-
// shift values by one dimension and stores date in second column
124-
$result['data'] = array_map(function ($tag) {
125-
$columns = count($tag);
126-
return array($tag[$columns - 2], $tag[$columns - 2], $tag[$columns - 1]);
127-
}, $result['data']);
128-
$result['data'] = $this->replaceDimension($result['data'], 1, date("Y-m-d H:i:s"));
124+
try {
125+
$result = $this->getDatasources()[$datasourceId]->readData($option);
126+
127+
if (isset($option['timestamp']) and $option['timestamp'] === 'true') {
128+
// if datasource should be timestamped/snapshoted
129+
// shift values by one dimension and stores date in second column
130+
$result['data'] = array_map(function ($tag) {
131+
$columns = count($tag);
132+
return array($tag[$columns - 2], $tag[$columns - 2], $tag[$columns - 1]);
133+
}, $result['data']);
134+
$result['data'] = $this->replaceDimension($result['data'], 1, date("Y-m-d H:i:s"));
135+
}
136+
} catch (\Error $e) {
137+
$result['error'] = $e->getMessage();
129138
}
130139
return $result;
131140
}
@@ -147,6 +156,7 @@ private function getOwnDatasources()
147156
{
148157
$datasources = [];
149158
$datasources[self::DATASET_TYPE_INTERNAL_FILE] = $this->FileService;
159+
$datasources[self::DATASET_TYPE_EXCEL] = $this->ExcelService;
150160
$datasources[self::DATASET_TYPE_GIT] = $this->GithubService;
151161
$datasources[self::DATASET_TYPE_EXTERNAL_FILE] = $this->ExternalFileService;
152162
$datasources[self::DATASET_TYPE_REGEX] = $this->RegexService;

‎lib/Controller/OutputController.php

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ private function getData($datasetMetadata)
114114
$option['user_id'] = $datasetMetadata['user_id'];
115115

116116
$result = $this->DatasourceController->read($datasourceId, $option);
117-
unset($result['error']);
117+
//unset($result['error']);
118118
}
119119

120120
$result['thresholds'] = $this->ThresholdService->read($datasetMetadata['id']);

‎lib/Datasource/Excel.php

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
<?php
2+
/**
3+
* Analytics
4+
*
5+
* This file is licensed under the Affero General Public License version 3 or
6+
* later. See the LICENSE.md file.
7+
*
8+
* @author Marcel Scherello <[email protected]>
9+
* @copyright 2021 Marcel Scherello
10+
*/
11+
12+
namespace OCA\Analytics\Datasource;
13+
14+
use OCP\Files\IRootFolder;
15+
use OCP\Files\NotFoundException;
16+
use OCP\IL10N;
17+
use OCP\ILogger;
18+
use PhpOffice\PhpSpreadsheet\IOFactory;
19+
use PhpOffice\PhpSpreadsheet\Reader\Exception;
20+
21+
class Excel implements IDatasource
22+
{
23+
private $logger;
24+
private $rootFolder;
25+
private $userId;
26+
private $l10n;
27+
28+
public function __construct(
29+
$userId,
30+
IL10N $l10n,
31+
ILogger $logger,
32+
IRootFolder $rootFolder
33+
)
34+
{
35+
$this->userId = $userId;
36+
$this->l10n = $l10n;
37+
$this->logger = $logger;
38+
$this->rootFolder = $rootFolder;
39+
}
40+
41+
/**
42+
* @return string Display Name of the datasource
43+
*/
44+
public function getName(): string
45+
{
46+
return $this->l10n->t('Local file: Excel');
47+
}
48+
49+
/**
50+
* @return int digit unique datasource id
51+
*/
52+
public function getId(): int
53+
{
54+
return 7;
55+
}
56+
57+
/**
58+
* @return array available options of the datasoure
59+
*/
60+
public function getTemplate(): array
61+
{
62+
$template = array();
63+
array_push($template, ['id' => 'link', 'name' => 'Filelink', 'placeholder' => 'filelink']);
64+
array_push($template, ['id' => 'sheet', 'name' => $this->l10n->t('Sheet'), 'placeholder' => $this->l10n->t('sheet name')]);
65+
array_push($template, ['id' => 'range', 'name' => $this->l10n->t('Cell range'), 'placeholder' => $this->l10n->t('e.g. A1:C3,A5:C5')]);
66+
return $template;
67+
}
68+
69+
/**
70+
* Read the Data
71+
* @param $option
72+
* @return array available options of the datasoure
73+
* @throws NotFoundException
74+
* @throws \OCP\Files\NotPermittedException
75+
* @throws Exception
76+
*/
77+
public function readData($option): array
78+
{
79+
80+
include_once __DIR__ . '/../../vendor/autoload.php';
81+
$header = $dataClean = $data = array();
82+
$headerrow = $errorMessage = 0;
83+
84+
$file = $this->rootFolder->getUserFolder($this->userId)->get($option['link']);
85+
$filePath = explode('/', ltrim($file->getPath(), '/'));
86+
// remove leading username
87+
array_shift($filePath);
88+
$filePath = implode('/', $filePath);
89+
$fileName = $file->getStorage()->getLocalFile($filePath);
90+
91+
$inputFileType = IOFactory::identify($fileName);
92+
$reader = IOFactory::createReader($inputFileType);
93+
$reader->setReadDataOnly(true);
94+
if (strlen($option['sheet']) > 0) {
95+
$reader->setLoadSheetsOnly([$option['sheet']]);
96+
}
97+
98+
$spreadsheet = $reader->load($fileName);
99+
100+
// separated columns can be selected via ranges e.g. "A1:B9, C1:C9"
101+
// these ranges are read and linked
102+
$ranges = str_getcsv($option['range'], ',');
103+
foreach ($ranges as $range) {
104+
$values = $spreadsheet->getActiveSheet()->rangeToArray(
105+
$range, // The worksheet range that we want to retrieve
106+
NULL, // Value that should be returned for empty cells
107+
TRUE, // Should formulas be calculated (the equivalent of getCalculatedValue() for each cell)
108+
TRUE, // Should values be formatted (the equivalent of getFormattedValue() for each cell)
109+
FALSE // Should the array be indexed by cell row and cell column
110+
);
111+
if (empty($data)) {
112+
// first range will fill the array with all rows
113+
$data = $values;
114+
} else {
115+
// further columns will be attatched to the first ones
116+
foreach ($data as $key => $value) {
117+
$data[$key] = array_merge($data[$key], $values[$key]);
118+
}
119+
120+
}
121+
}
122+
123+
foreach ($data as $key => $value) {
124+
if ($headerrow === 0) {
125+
$header = array_values($value);
126+
$headerrow = 1;
127+
} else if (!$this->containsOnlyNull($value)) {
128+
array_push($dataClean, array_values($value));
129+
}
130+
}
131+
132+
return [
133+
'header' => $header,
134+
'data' => $dataClean,
135+
'error' => $errorMessage,
136+
];
137+
}
138+
139+
private function containsOnlyNull($array)
140+
{
141+
return array_reduce($array, function ($carry, $item) {
142+
return $carry += (is_null($item) ? 0 : 1);
143+
}, 0) > 0 ? false : true;
144+
}
145+
}

0 commit comments

Comments
 (0)