Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add extra status verification before automation start #135

Merged
merged 9 commits into from
Jan 25, 2024
Merged

add extra status verification before automation start #135

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
90bf1ee
feature(STFClient): do not use 'unheathy' and 'unauthorized' devices …
akamarouski Jan 11, 2024
5a2001e
feature(STFClient): mark device as unheathy if response status 0 (whe…
akamarouski Jan 12, 2024
50fa552
feature(MobileRemoteProxy): add more info about 'Node is down' error
akamarouski Jan 12, 2024
74d4ddd
refactor(MobileRemoteProxy): code cleanup
akamarouski Jan 24, 2024
c0daacc
refactor(STFClient): change log
akamarouski Jan 24, 2024
e91375c
feature(STFClient): add ignore timeouts to the ENV variables
akamarouski Jan 25, 2024
7f995eb
refactor(STFClient): change minutes to seconds in logs
akamarouski Jan 25, 2024
a720472
refactor(MobileRemoteProxy): modify 'Node is down' logs
akamarouski Jan 25, 2024
aa5695f
feature: check is there device with udid in STF
akamarouski Jan 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 39 additions & 1 deletion src/main/java/com/zebrunner/mcloud/grid/MobileRemoteProxy.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*******************************************************************************/
package com.zebrunner.mcloud.grid;

import java.net.URL;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
Expand All @@ -28,7 +29,9 @@
import org.apache.http.entity.ContentType;
import org.apache.http.entity.StringEntity;
import org.openqa.grid.common.RegistrationRequest;
import org.openqa.grid.common.exception.GridException;
import org.openqa.grid.internal.GridRegistry;
import org.openqa.grid.internal.RemoteProxy;
import org.openqa.grid.internal.TestSession;
import org.openqa.grid.internal.TestSlot;
import org.openqa.grid.selenium.proxy.DefaultRemoteProxy;
Expand All @@ -55,6 +58,20 @@ public class MobileRemoteProxy extends DefaultRemoteProxy {

public MobileRemoteProxy(RegistrationRequest request, GridRegistry registry) {
super(request, registry);
getTestSlots().stream()
.findAny()
.ifPresent(slot -> {
String udid = String.valueOf(CapabilityUtils.getAppiumCapability(slot.getCapabilities(), "udid")
.orElse(""));
if (StringUtils.isBlank(udid)) {
throw new GridException(String.format("Appium node must have 'UDID' capability. Slot capabilities: %s",
slot.getCapabilities()));
}
if (!STFClient.isDevicePresentInSTF(udid)) {
throw new GridException(String.format("Could not find device with udid '%s' in STF. Slot capabilities: %s",
udid, slot.getCapabilities()));
}
});
}

@Override
Expand All @@ -68,7 +85,28 @@ public TestSession getNewSession(Map<String, Object> requestedCapability) {
return null;
}
if (isDown()) {
LOGGER.info(() -> "Node is down.");
getTestSlots().stream()
.findAny()
.ifPresent((slot -> {
LOGGER.info(() -> String.format("Node is down: '[%s]-'%s:%s' (%s)'",
CapabilityUtils.getAppiumCapability(slot.getCapabilities(), "deviceName")
.orElse(StringUtils.EMPTY),
Optional.of(slot)
.map(TestSlot::getProxy)
.map(RemoteProxy::getRemoteHost)
.map(URL::getHost)
.orElse(StringUtils.EMPTY),
Optional.of(slot)
.map(TestSlot::getProxy)
.map(RemoteProxy::getRemoteHost)
.map(URL::getPort)
.map(String::valueOf)
.orElse(StringUtils.EMPTY),
CapabilityUtils.getAppiumCapability(slot.getCapabilities(), "udid")
.orElse(StringUtils.EMPTY)
)
);
}));
return null;
}
if (!hasCapability(requestedCapability)) {
Expand Down
86 changes: 84 additions & 2 deletions src/main/java/com/zebrunner/mcloud/grid/integration/client/STFClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
*******************************************************************************/
package com.zebrunner.mcloud.grid.integration.client;

import java.time.Duration;
import java.util.HashMap;
import java.util.Map;
import java.util.Optional;
Expand All @@ -36,11 +37,29 @@
public final class STFClient {
private static final Logger LOGGER = Logger.getLogger(STFClient.class.getName());
private static final Map<String, STFClient> STF_CLIENTS = new ConcurrentHashMap<>();
private static final Map<String, Duration> STF_DEVICE_IGNORE_AUTOMATION_TIMERS = new ConcurrentHashMap<>();
private static final String STF_URL = System.getenv("STF_URL");
private static final String DEFAULT_STF_TOKEN = System.getenv("STF_TOKEN");
// Max time is seconds for reserving devices in STF
private static final String DEFAULT_STF_TIMEOUT = System.getenv("STF_TIMEOUT");

private static final Duration INVALID_STF_RESPONSE_TIMEOUT = Optional.ofNullable(System.getenv("STF_DEVICE_INVALID_RESPONSE_IGNORE_TIMEOUT"))
.filter(StringUtils::isNotBlank)
.map(Integer::parseInt)
.map(Duration::ofSeconds)
.orElse(Duration.ofMinutes(10));
private static final Duration UNAUTHORIZED_TIMEOUT = Optional.ofNullable(System.getenv("STF_DEVICE_UNAUTHORIZED_IGNORE_TIMEOUT"))
.filter(StringUtils::isNotBlank)
.map(Integer::parseInt)
.map(Duration::ofSeconds)
.orElse(Duration.ofMinutes(10));

private static final Duration UNHEALTHY_TIMEOUT = Optional.ofNullable(System.getenv("STF_DEVICE_UNHEALTHY_IGNORE_TIMEOUT"))
.filter(StringUtils::isNotBlank)
.map(Integer::parseInt)
.map(Duration::ofSeconds)
.orElse(Duration.ofMinutes(5));

private Platform platform;
private String token;
private boolean isReservedManually = false;
Expand Down Expand Up @@ -68,10 +87,24 @@ public static STFDevice getSTFDevice(String udid) {
* Reserve STF device
*/
public static boolean reserveSTFDevice(String deviceUDID, Map<String, Object> requestedCapabilities, String sessionUUID) {
LOGGER.info(() -> String.format("[STF-%s] Reserve STF Device.", sessionUUID));
if (!isSTFEnabled()) {
return true;
}
LOGGER.info(() -> String.format("[STF-%s] Reserve STF Device.", sessionUUID));

if (STF_DEVICE_IGNORE_AUTOMATION_TIMERS.get(deviceUDID) != null) {
Duration timeout = STF_DEVICE_IGNORE_AUTOMATION_TIMERS.get(deviceUDID);
if (Duration.ofMillis(System.currentTimeMillis()).compareTo(timeout) < 0) {
LOGGER.warning(() -> String.format("[STF-%s] The next attempt to reserve '%s' STF device will be given only after '%s' seconds.",
sessionUUID,
deviceUDID,
timeout.minus(Duration.ofMillis(System.currentTimeMillis())).toSeconds()));
return false;
} else {
STF_DEVICE_IGNORE_AUTOMATION_TIMERS.remove(deviceUDID);
}
}

if (STF_CLIENTS.get(deviceUDID) != null) {
LOGGER.warning(() -> String.format("Device '%s' already busy (in the local pool). Info: %s", deviceUDID, STF_CLIENTS.get(deviceUDID)));
}
Expand Down Expand Up @@ -116,10 +149,31 @@ public static boolean reserveSTFDevice(String deviceUDID, Map<String, Object> re
}

STFDevice stfDevice = optionalSTFDevice.get();
LOGGER.info(() -> String.format("[STF-%s] STF Device info: %s", sessionUUID, stfDevice));
LOGGER.info(() -> String.format("[STF-%s] STF device info: %s", sessionUUID, stfDevice));

stfClient.setDevice(stfDevice);

if (stfDevice.getStatus() == null) {
LOGGER.warning(() -> String.format("[STF-%s] STF device status is null. It will be ignored: %s seconds.", sessionUUID,
INVALID_STF_RESPONSE_TIMEOUT.toSeconds()));
STF_DEVICE_IGNORE_AUTOMATION_TIMERS.put(deviceUDID, Duration.ofMillis(System.currentTimeMillis()).plus(INVALID_STF_RESPONSE_TIMEOUT));
return false;
}

if (stfDevice.getStatus().intValue() == 2) {
LOGGER.warning(() -> String.format("[STF-%s] STF device status 'UNAUTHORIZED'. It will be ignored: %s seconds.", sessionUUID,
UNAUTHORIZED_TIMEOUT.toSeconds()));
STF_DEVICE_IGNORE_AUTOMATION_TIMERS.put(deviceUDID, Duration.ofMillis(System.currentTimeMillis()).plus(UNAUTHORIZED_TIMEOUT));
return false;
}

if (stfDevice.getStatus() == 7) {
LOGGER.warning(() -> String.format("[STF-%s] STF device status 'UNHEALTHY'. It will be ignored: %s seconds.", sessionUUID,
UNHEALTHY_TIMEOUT.toSeconds()));
STF_DEVICE_IGNORE_AUTOMATION_TIMERS.put(deviceUDID, Duration.ofMillis(System.currentTimeMillis()).plus(UNHEALTHY_TIMEOUT));
return false;
}

if (stfDevice.getOwner() != null && StringUtils.equals(stfDevice.getOwner().getName(), user.getObject().getUser().getName()) &&
stfDevice.getPresent() &&
stfDevice.getReady()) {
Expand All @@ -137,6 +191,17 @@ public static boolean reserveSTFDevice(String deviceUDID, Map<String, Object> re
if (response.getStatus() != 200) {
LOGGER.warning(() -> String.format("[STF-%s] Could not reserve STF device with udid: %s. Status: %s. Response: %s",
sessionUUID, deviceUDID, response.getStatus(), response.getObject()));
if (response.getStatus() == 0) {
LOGGER.warning(() -> String.format("[STF-%s] Device will be marked as unhealthy due to response status '0'.", sessionUUID));
entity.put("body", Map.of("status", "Unhealthy"));
HttpClient.Response r = HttpClient.uri(Path.STF_DEVICES_ITEM_PATH, STF_URL, deviceUDID)
.withAuthorization(buildAuthToken(stfToken))
.put(Void.class, entity);
if (r.getStatus() != 200) {
LOGGER.warning(() -> String.format("[STF-%s] Could not mark device as unhealthy. Status: %s. Response: %s", sessionUUID,
r.getStatus(), r.getObject()));
}
}
return false;
}
} else {
Expand Down Expand Up @@ -311,6 +376,23 @@ private static boolean isSTFEnabled() {
return (!StringUtils.isEmpty(STF_URL) && !StringUtils.isEmpty(DEFAULT_STF_TOKEN));
}

public static boolean isDevicePresentInSTF(String udid) {
if (!isSTFEnabled()) {
return true;
}
HttpClient.Response<Devices> devices = HttpClient.uri(Path.STF_DEVICES_PATH, STF_URL)
.withAuthorization(buildAuthToken(DEFAULT_STF_TOKEN))
.get(Devices.class);
if (devices.getStatus() != 200) {
LOGGER.warning(() -> String.format("[NODE REGISTRATION] Unable to get devices status. HTTP status: %s", devices.getStatus()));
return false;
}
return devices.getObject()
.getDevices()
.stream()
.anyMatch(device -> StringUtils.equals(device.getSerial(), udid));
}

@Override public String toString() {
return "STFClient{" +
"platform=" + platform +
Expand Down