Commit 50611e0d authored by To-om's avatar To-om

Merge branch 'release/3.3.0-RC3'

parents 7a25bcf0 40135202
......@@ -159,6 +159,7 @@ steps:
strip_components: 3
when:
branch: [develop]
event: {exclude: [pull_request]}
- name: deploy binaries in integration environment
image: appleboy/drone-ssh
......@@ -170,6 +171,7 @@ steps:
- ./start thehive ${DRONE_BUILD_NUMBER}
when:
branch: [develop]
event: {exclude: [pull_request]}
# Deploy binaries in staging environment
- name: copy binaries in staging environment
......@@ -183,6 +185,7 @@ steps:
strip_components: 3
when:
branch: [master]
event: {exclude: [pull_request]}
- name: deploy binaries in staging environment
image: appleboy/drone-ssh
......@@ -194,6 +197,7 @@ steps:
- ./start thehive ${DRONE_BUILD_NUMBER}
when:
branch: [master]
event: {exclude: [pull_request]}
volumes:
- name: cache
......
# Change Log
## [3.3.0-RC2](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC2) (2019-02-07)
## [3.3.0-RC3](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC3) (2019-02-21)
[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.0-RC2...3.3.0-RC3)
**Implemented enhancements:**
- Add a UI configuration admin section [\#888](https://github.com/TheHive-Project/TheHive/issues/888)
- Add a Related Alerts link to case details view [\#884](https://github.com/TheHive-Project/TheHive/issues/884)
- Update Copyright with year 2019 [\#879](https://github.com/TheHive-Project/TheHive/issues/879)
- Provide a quick link to copy alert id [\#870](https://github.com/TheHive-Project/TheHive/issues/870)
- \[BUG\] Audit trail for alert ignore [\#863](https://github.com/TheHive-Project/TheHive/issues/863)
- Related artifacts: IOC/not IOC [\#838](https://github.com/TheHive-Project/TheHive/issues/838)
- Feature: Add "auto-completion" to the UI [\#831](https://github.com/TheHive-Project/TheHive/issues/831)
- Improvement: Upload of observables seem to fail "silently" [\#829](https://github.com/TheHive-Project/TheHive/issues/829)
- Feature Request: link to and from Hive to MISP [\#820](https://github.com/TheHive-Project/TheHive/issues/820)
- Disable clickable widgets in dashboard edit mode [\#485](https://github.com/TheHive-Project/TheHive/issues/485)
- Ability to disable "New Case" -\> "Empty case" [\#449](https://github.com/TheHive-Project/TheHive/issues/449)
**Fixed bugs:**
- Drone build fails on pull-requests [\#882](https://github.com/TheHive-Project/TheHive/issues/882)
- AKKA version missmatch [\#877](https://github.com/TheHive-Project/TheHive/issues/877)
- Label Typo in Updated Alerts [\#874](https://github.com/TheHive-Project/TheHive/issues/874)
- Log message related to MISP synchronization is confusing [\#871](https://github.com/TheHive-Project/TheHive/issues/871)
- Cortex responders with DataType `thehive:case\_artifact` do not show up within thehive when attempting to run them for observables. [\#869](https://github.com/TheHive-Project/TheHive/issues/869)
- Alert updates and tracking \(follow\) [\#856](https://github.com/TheHive-Project/TheHive/issues/856)
**Merged pull requests:**
- Update akka version [\#878](https://github.com/TheHive-Project/TheHive/pull/878) ([zpriddy](https://github.com/zpriddy))
- Fix Update Label to Warning [\#873](https://github.com/TheHive-Project/TheHive/pull/873) ([zpriddy](https://github.com/zpriddy))
## [3.3.0-RC2](https://github.com/TheHive-Project/TheHive/tree/3.3.0-RC2) (2019-02-07)
[Full Changelog](https://github.com/TheHive-Project/TheHive/compare/3.3.0-RC1...3.3.0-RC2)
**Fixed bugs:**
......
......@@ -21,7 +21,7 @@ object Dependencies {
val reflections = "org.reflections" % "reflections" % "0.9.11"
val zip4j = "net.lingala.zip4j" % "zip4j" % "1.3.2"
val elastic4play = "org.thehive-project" %% "elastic4play" % "1.8.0-1"
val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.11"
val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.11"
val akkaCluster = "com.typesafe.akka" %% "akka-cluster" % "2.5.19"
val akkaClusterTools = "com.typesafe.akka" %% "akka-cluster-tools" % "2.5.19"
}
}
......@@ -38,31 +38,35 @@ class ArtifactCtrl @Inject() (
private[ArtifactCtrl] lazy val logger = Logger(getClass)
// extract a file from the archive and make sure its size matches the header (to protect against zip bombs)
private def extractAndCheckSize(zipFile: ZipFile, header: FileHeader)(implicit authContext: AuthContext): FileInputValue = {
val file = tempSrv.newTemporaryFile(header.getFileName, "-fromZipFile")
val input = zipFile.getInputStream(header)
val size = header.getUncompressedSize
val sizedInput: FilterInputStream = new FilterInputStream(input) {
var totalRead = 0
override def read(): Int = {
if (totalRead < size) {
totalRead += 1
super.read()
private def extractAndCheckSize(zipFile: ZipFile, header: FileHeader)(implicit authContext: AuthContext): Option[FileInputValue] = {
val fileName = header.getFileName
if (fileName.contains('/')) None
else {
val file = tempSrv.newTemporaryFile(fileName, "-fromZipFile")
val input = zipFile.getInputStream(header)
val size = header.getUncompressedSize
val sizedInput: FilterInputStream = new FilterInputStream(input) {
var totalRead = 0
override def read(): Int = {
if (totalRead < size) {
totalRead += 1
super.read()
}
else throw BadRequestError("Error extracting file: output size doesn't match header")
}
else throw BadRequestError("Error extracting file: output size doesn't match header")
}
Files.delete(file)
val fileSize = Files.copy(sizedInput, file)
if (fileSize != size) {
file.toFile.delete()
throw InternalError("Error extracting file: output size doesn't match header")
}
input.close()
val contentType = Option(Files.probeContentType(file)).getOrElse("application/octet-stream")
Some(FileInputValue(header.getFileName, file, contentType))
}
Files.delete(file)
val fileSize = Files.copy(sizedInput, file)
if (fileSize != size) {
file.toFile.delete()
throw InternalError("Error extracting file: output size doesn't match header")
}
input.close()
val contentType = Option(Files.probeContentType(file)).getOrElse("application/octet-stream")
FileInputValue(header.getFileName, file, contentType)
}
@Timed
......@@ -91,7 +95,7 @@ class ArtifactCtrl @Inject() (
}
val multiFields = files.filterNot(_.isDirectory)
.map(extractAndCheckSize(zipFile, _))
.flatMap(extractAndCheckSize(zipFile, _))
.map { fiv
fields
.unset("isZip")
......
package models
import java.util.Date
import javax.inject.{ Inject, Provider, Singleton }
import javax.inject.{ Inject, Provider, Singleton }
import scala.concurrent.{ ExecutionContext, Future }
import scala.util.Success
......@@ -12,6 +12,7 @@ import play.api.libs.json.JsValue.jsValueToJsLookup
import play.api.libs.json.Json.toJsFieldJsValueWrapper
import play.api.libs.json._
import akka.stream.scaladsl.Sink
import akka.stream.{ IOResult, Materializer }
import akka.{ Done, NotUsed }
import models.JsonFormat.artifactStatusFormat
......@@ -114,9 +115,11 @@ class ArtifactModel @Inject() (
override def getStats(entity: BaseEntity): Future[JsObject] = {
entity match {
case artifact: Artifact
val (_, total) = artifactSrv.get.findSimilar(artifact, Some("0-0"), Nil)
total.failed.foreach(t logger.error("Artifact.getStats error", t))
total.map { t Json.obj("seen" t) }
val (similarArtifacts, total) = artifactSrv.get.findSimilar(artifact, Some("0-1"), Seq("-ioc"))
for {
ioc similarArtifacts.runWith(Sink.headOption).map(_.fold(false)(_.ioc()))
t total
} yield Json.obj("seen" t, "ioc" ioc)
case _ Future.successful(JsObject.empty)
}
}
......
......@@ -146,7 +146,7 @@ class AlertSrv(
def update(alert: Alert, fields: Fields, modifyConfig: ModifyConfig)(implicit authContext: AuthContext): Future[Alert] = {
val follow = fields.getBoolean("follow").getOrElse(alert.follow())
val newStatus = if (follow) AlertStatus.Updated else alert.status()
val newStatus = if (follow && alert.status() != AlertStatus.New) AlertStatus.Updated else alert.status()
val updatedAlert = updateSrv(alert, fields.set("status", Json.toJson(newStatus)), modifyConfig)
alert.caze() match {
case Some(caseId) if follow
......
......@@ -139,6 +139,18 @@ class ActionOperationSrv @Inject() (
}
}
def findArtifactEntity(entity: BaseEntity): Future[Artifact] = {
import org.elastic4play.services.QueryDSL._
(entity, entity.model) match {
case (a: Artifact, _) Future.successful(a)
case (_, model: ChildModelDef[_, _, _, _])
findSrv(model.parentModel, "_id" ~= entity.parentId.getOrElse(throw InternalError(s"Child entity $entity has no parent ID")), Some("0-1"), Nil)
._1.runWith(Sink.head).flatMap(findArtifactEntity _)
case _ Future.failed(BadRequestError("Artifact not found"))
}
}
def execute(entity: BaseEntity, operation: ActionOperation)(implicit authContext: AuthContext): Future[ActionOperation] = {
if (operation.status == ActionOperationStatus.Waiting) {
Retry()(classOf[VersionConflictEngineException]) {
......
......@@ -72,10 +72,14 @@ class CortexActionSrv @Inject() (
def findResponderFor(entityType: String, entityId: String): Future[Seq[Responder]] = {
for {
(tlp, pap) getEntity(entityType, entityId)
.flatMap(actionOperationSrv.findCaseEntity)
.map { caze (caze.tlp(), caze.pap()) }
.recover { case _ (0L, 0L) }
entity getEntity(entityType, entityId)
artifactTlp actionOperationSrv
.findArtifactEntity(entity)
.map(a Some(a.tlp()))
.recover { case _ None }
(tlp, pap) actionOperationSrv.findCaseEntity(entity)
.map { caze (artifactTlp.getOrElse(caze.tlp()), caze.pap()) }
.recover { case _ (artifactTlp.getOrElse(0L), 0L) }
query = Json.obj(
"dataTypeList" s"thehive:$entityType")
responders findResponders(query)
......
......@@ -106,11 +106,13 @@ case class MispConnection(
"name" name,
"version" version,
"status" "OK",
"url" baseUrl,
"purpose" purpose.toString)
case None Json.obj(
"name" name,
"version" "",
"status" "ERROR",
"url" baseUrl,
"purpose" purpose.toString)
}
}
......
......@@ -134,9 +134,10 @@ class MispSynchro @Inject() (
}
def synchronize(mispConnection: MispConnection, lastSyncDate: Option[Date])(implicit authContext: AuthContext): Source[Try[Alert], NotUsed] = {
logger.info(s"Synchronize MISP ${mispConnection.name} from $lastSyncDate")
val syncFrom = mispConnection.syncFrom(lastSyncDate.getOrElse(new Date(0)))
logger.info(s"Last synchronization of MISP ${mispConnection.name} is ${lastSyncDate.fold("Never")(_.toString)}, synchronize from $syncFrom")
// get events that have been published after the last synchronization
mispSrv.getEventsFromDate(mispConnection, mispConnection.syncFrom(lastSyncDate.getOrElse(new Date(0))))
mispSrv.getEventsFromDate(mispConnection, syncFrom)
// get related alert
.mapAsyncUnordered(1) { event
logger.trace(s"Looking for alert misp:${event.source}:${event.sourceRef}")
......
......@@ -144,11 +144,13 @@
<script src="scripts/controllers/admin/AdminMetricsCtrl.js"></script>
<script src="scripts/controllers/admin/AdminObservablesCtrl.js"></script>
<script src="scripts/controllers/admin/AdminReportTemplatesCtrl.js"></script>
<script src="scripts/controllers/admin/AdminUiSettingsCtrl.js"></script>
<script src="scripts/controllers/admin/AdminUserDialogCtrl.js"></script>
<script src="scripts/controllers/admin/AdminUsersCtrl.js"></script>
<script src="scripts/controllers/alert/AlertEventCtrl.js"></script>
<script src="scripts/controllers/alert/AlertListCtrl.js"></script>
<script src="scripts/controllers/alert/AlertStatsCtrl.js"></script>
<script src="scripts/controllers/case/CaseAlertsCtrl.js"></script>
<script src="scripts/controllers/case/CaseCloseModalCtrl.js"></script>
<script src="scripts/controllers/case/CaseCreationCtrl.js"></script>
<script src="scripts/controllers/case/CaseDeleteModalCtrl.js"></script>
......@@ -268,6 +270,7 @@
<script src="scripts/services/StreamStatSrv.js"></script>
<script src="scripts/services/TagSrv.js"></script>
<script src="scripts/services/TaskLogSrv.js"></script>
<script src="scripts/services/UiSettingsSrv.js"></script>
<script src="scripts/services/UserInfoSrv.js"></script>
<script src="scripts/services/UserSrv.js"></script>
<script src="scripts/services/UtilsSrv.js"></script>
......
......@@ -34,8 +34,8 @@ angular.module('thehive', ['ngAnimate', 'ngMessages', 'ngSanitize', 'ui.bootstra
templateUrl: 'views/login.html',
resolve: {
appConfig: function(VersionSrv) {
return VersionSrv.get();
}
return VersionSrv.get();
}
},
params: {
autoLogin: false
......@@ -77,6 +77,10 @@ angular.module('thehive', ['ngAnimate', 'ngMessages', 'ngSanitize', 'ui.bootstra
appLayout: function($q, $rootScope, AppLayoutSrv) {
AppLayoutSrv.init();
return $q.resolve();
},
uiConfig: function($q, UiSettingsSrv) {
UiSettingsSrv.all();
return $q.resolve();
}
}
})
......@@ -215,6 +219,18 @@ angular.module('thehive', ['ngAnimate', 'ngMessages', 'ngSanitize', 'ui.bootstra
controller: 'AdminObservablesCtrl',
title: 'Observable administration'
})
.state('app.administration.ui-settings', {
url: '/ui-settings',
templateUrl: 'views/partials/admin/ui-settings.html',
controller: 'AdminUiSettingsCtrl',
controllerAs: '$vm',
title: 'UI settings',
resolve: {
uiConfig: function(UiSettingsSrv) {
return UiSettingsSrv.all();
}
}
})
.state('app.case', {
abstract: true,
url: 'case/{caseId}',
......@@ -263,6 +279,20 @@ angular.module('thehive', ['ngAnimate', 'ngMessages', 'ngSanitize', 'ui.bootstra
templateUrl: 'views/partials/case/case.links.html',
controller: 'CaseLinksCtrl'
})
.state('app.case.alerts', {
url: '/alerts',
templateUrl: 'views/partials/case/case.alerts.html',
controller: 'CaseAlertsCtrl',
resolve: {
alerts: function($stateParams, CaseSrv) {
return CaseSrv.alerts({range: 'all'}, {
query: {
case: $stateParams.caseId
}
}).$promise;
}
}
})
.state('app.case.tasks-item', {
url: '/tasks/{itemId}',
templateUrl: 'views/partials/case/case.tasks.item.html',
......@@ -299,6 +329,20 @@ angular.module('thehive', ['ngAnimate', 'ngMessages', 'ngSanitize', 'ui.bootstra
resolve: {
appConfig: function(VersionSrv) {
return VersionSrv.get();
},
artifact: function($q, $stateParams, CaseArtifactSrv, NotificationSrv) {
var deferred = $q.defer();
CaseArtifactSrv.api().get({
'artifactId': $stateParams.itemId
}).$promise.then(function(data) {
deferred.resolve(data);
}).catch(function(response) {
deferred.reject(response);
NotificationSrv.error('Observable Details', response.data, response.status);
});
return deferred.promise;
}
}
})
......@@ -433,7 +477,7 @@ angular.module('thehive', ['ngAnimate', 'ngMessages', 'ngSanitize', 'ui.bootstra
var renderer = defaults.renderer;
var linkRenderer = _.wrap(renderer.link, function(originalLink, href, title, text) {
var html = originalLink.call(renderer, href, title, text);
return html.replace(/^<a /, '<a target="_blank" rel="nofollow" ')
return html.replace(/^<a /, '<a target="_blank" rel="nofollow" ');
});
// Customize the link renderer
......
......@@ -179,13 +179,16 @@ angular.module('theHiveControllers').controller('RootCtrl',
resolve: {
templates: function(){
return $scope.templates;
},
uiSettings: function(UiSettingsSrv) {
return UiSettingsSrv.all();
}
}
});
modal.result.then(function(template) {
$scope.createNewCase(template);
})
});
};
$scope.aboutTheHive = function() {
......
......@@ -78,11 +78,11 @@
self.clearUniqueReferenceError = function(form) {
form.reference.$setValidity('unique', true);
form.reference.$setPristine();
}
};
self.cancel = function() {
$uibModalInstance.dismiss();
}
};
self.onNamechanged = function(form) {
if (!self.customField.name) {
......
......@@ -59,12 +59,12 @@
}
});
modalInstance.result.then(function(data) {
modalInstance.result.then(function(/*data*/) {
self.initCustomfields();
CustomFieldsCacheSrv.clearCache();
$scope.$emit('custom-fields:refresh');
});
}
};
self.initCustomfields();
});
......
(function() {
'use strict';
angular.module('theHiveControllers').controller('AdminUiSettingsCtrl', function($scope, $q, NotificationSrv, UiSettingsSrv, uiConfig) {
var self = this;
self.isDirtySetting = function(key, newValue) {
var currentValue = (self.currentSettings[key] || {}).value;
return newValue !== currentValue;
};
self.save = function(/*form*/) {
var promises = [];
self.settingsKeys.forEach(function(key) {
if(self.isDirtySetting(key, self.configs[key])) {
if(!self.currentSettings[key]) {
promises.push(UiSettingsSrv.create(key, self.configs[key]));
} else {
promises.push(UiSettingsSrv.update(self.currentSettings[key].id, key, self.configs[key]));
}
}
});
if(promises.length === 0) {
return;
}
$q.all(promises)
.then(function(/*responses*/) {
self.loadSettings();
NotificationSrv.log('UI Settings updated successfully', 'success');
})
.catch(function(/*errors*/) {
NotificationSrv.error('An error occurred during UI Settings update');
});
};
self.loadSettings = function(configurations) {
var notifyRoot = false;
var promise;
if(configurations) {
promise = $q.resolve(configurations);
} else {
promise = UiSettingsSrv.all(true);
notifyRoot = true;
}
promise.then(function(configs) {
self.settingsKeys = UiSettingsSrv.keys;
self.currentSettings = configs;
self.configs = {};
self.settingsKeys.forEach(function(key) {
self.configs[key] = (configs[key] || {}).value;
});
if(notifyRoot) {
$scope.$emit('ui-settings:refresh', configs);
}
});
};
self.loadSettings(uiConfig);
});
})();
(function() {
'use strict';
angular.module('theHiveControllers')
.controller('AlertEventCtrl', function($scope, $rootScope, $state, $uibModal, $uibModalInstance, CustomFieldsCacheSrv, CaseResolutionStatus, AlertingSrv, NotificationSrv, event, templates) {
.controller('AlertEventCtrl', function($scope, $rootScope, $state, $uibModal, $uibModalInstance, CustomFieldsCacheSrv, CaseResolutionStatus, AlertingSrv, NotificationSrv, clipboard, event, templates) {
var self = this;
var eventId = event.id;
......@@ -254,6 +254,10 @@
}
};
self.copyId = function(id) {
clipboard.copyText(id);
};
self.load();
});
})();
(function() {
'use strict';
angular.module('theHiveControllers')
.controller('AlertListCtrl', function($rootScope, $scope, $q, $state, $uibModal, TagSrv, CaseTemplateSrv, AlertingSrv, NotificationSrv, FilteringSrv, CortexSrv, Severity) {
.controller('AlertListCtrl', function($rootScope, $scope, $q, $state, $uibModal, TagSrv, CaseTemplateSrv, AlertingSrv, NotificationSrv, FilteringSrv, CortexSrv, Severity, VersionSrv) {
var self = this;
self.urls = VersionSrv.mispUrls();
self.list = [];
self.selection = [];
self.menu = {
......@@ -322,6 +324,9 @@
resolve: {
templates: function(){
return templates;
},
uiSettings: function(UiSettingsSrv) {
return UiSettingsSrv.all();
}
}
});
......
(function() {
'use strict';
angular.module('theHiveControllers').controller('CaseAlertsCtrl',
function($scope, $state, $stateParams, $uibModal, $timeout, CaseTabsSrv, VersionSrv, alerts) {
$scope.caseId = $stateParams.caseId;
$scope.alerts = alerts;
$scope.alertStats = [];
$scope.currentFilter = '';
$scope.filtering = {};
$scope.sorting = {
field: '-date'
};
var tabName = 'alerts-' + $scope.caseId;
$scope.mispUrls = VersionSrv.mispUrls();
// Add tab
CaseTabsSrv.addTab(tabName, {
name: tabName,
label: 'Related Alerts',
closable: true,
state: 'app.case.alerts',
params: {}
});
// Select tab
$timeout(function() {
CaseTabsSrv.activateTab(tabName);
}, 0);
$scope.initStats = function(data) {
var stats = {
type: {},
source: {}
};
// Init the stats object
_.each(data, function(item) {
stats.type[item.type] = stats.type[item.type] ? stats.type[item.type] + 1 : 1;
stats.source[item.source] = stats.source[item.source] ? stats.source[item.source] + 1 : 1;
});
var result = {};
Object.keys(stats).forEach(function(field) {
result[field] = [];
Object.keys(stats[field]).forEach(function(key) {
result[field].push({
key: key,
value: stats[field][key]
});
});
});
return result;
};
$scope.filterBy = function(field, filter) {
$scope.currentFilter = filter;
if(field === '') {
$scope.filtering = {};
} else {
var temp = {};
temp[field] = filter;
$scope.filtering = temp;
}
};
$scope.sortBy = function(field) {
if($scope.sorting.field.substr(1) !== field) {
$scope.sorting.field = '+' + field;
} else {
$scope.sorting.field = ($scope.sorting.field === '+' + field) ? '-'+field : '+'+field;
}
};
$scope.alertStats = $scope.initStats($scope.alerts);
}
);
})();
(function() {
'use strict';
angular.module('theHiveControllers').controller('CaseDetailsCtrl', function($scope, $state, $uibModal, CaseTabsSrv, UserInfoSrv, PSearchSrv) {
angular.module('theHiveControllers').controller('CaseDetailsCtrl', function($scope, $state, $uibModal, CaseTabsSrv, UserInfoSrv, TagSrv, PSearchSrv) {
CaseTabsSrv.activateTab($state.current.data.tab);
......@@ -98,6 +98,10 @@
itemId: attachment.case_task.id
});
};
$scope.getCaseTags = function(query) {
return TagSrv.fromCases(query);
};
});
angular.module('theHiveControllers').controller('CaseCustomFieldsCtrl', function($scope, $uibModal, CustomFieldsCacheSrv) {
......@@ -108,13 +112,13 @@
return {
name: name,
order: definition.order
}
};
}), function(item){
return item.order;
}), 'name');
return result;
}
};
$scope.getCustomFieldName = function(fieldDef) {
return 'customFields.' + fieldDef.reference + '.' + fieldDef.type;
......@@ -178,4 +182,4 @@
};
});
})();
\ No newline at end of file
})();
......@@ -132,6 +132,15 @@
field: 'status'
});
$scope.alerts = StreamStatSrv({
scope: $scope,
rootId: caseId,
query: { 'case': caseId },
result: {},
objectType: 'alert',
field: 'type'
});
$scope.$on('tasks:task-removed', function(event, task) {