Skip to content

kubernetes: Deploy kubernetes-provider when creating a cluster #5254

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
yadvr merged 5 commits into from
Aug 9, 2021
Merged

kubernetes: Deploy kubernetes-provider when creating a cluster #5254

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
961fdd2
kubernetes: Deploy kubernetes-provider when creating a cluster
davidjumani Jul 29, 2021
98015bf
Fix tests
davidjumani Jul 30, 2021
1db057b
Addressing comments
davidjumani Jul 30, 2021
6575a56
Update plugins/integrations/kubernetes-service/src/main/java/com/clou…
davidjumani Aug 2, 2021
e4ed9c5
Addressing comments
davidjumani Aug 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions api/src/main/java/com/cloud/user/AccountService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,4 +121,6 @@ UserAccount createUserAccount(String userName, String password, String firstName
UserAccount getUserAccountById(Long userId);

public Map<String, String> getKeys(GetUserKeysCmd cmd);

public Map<String, String> getKeys(Long userId);
}
51 changes: 50 additions & 1 deletion ...etes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterManagerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.UUID;

import javax.inject.Inject;
import javax.naming.ConfigurationException;
Expand All @@ -53,6 +54,7 @@
import org.apache.cloudstack.api.response.KubernetesClusterResponse;
import org.apache.cloudstack.api.response.ListResponse;
import org.apache.cloudstack.api.response.UserVmResponse;
import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.context.CallContext;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.ConfigKey;
Expand Down Expand Up @@ -134,7 +136,11 @@
import com.cloud.user.AccountManager;
import com.cloud.user.AccountService;
import com.cloud.user.SSHKeyPairVO;
import com.cloud.user.User;
import com.cloud.user.UserAccount;
import com.cloud.user.UserVO;
import com.cloud.user.dao.SSHKeyPairDao;
import com.cloud.user.dao.UserDao;
import com.cloud.utils.Pair;
import com.cloud.utils.Ternary;
import com.cloud.utils.component.ComponentContext;
Expand Down Expand Up @@ -198,6 +204,8 @@ public class KubernetesClusterManagerImpl extends ManagerBase implements Kuberne
@Inject
protected AccountManager accountManager;
@Inject
protected UserDao userDao;
@Inject
protected VMInstanceDao vmInstanceDao;
@Inject
protected UserVmJoinDao userVmJoinDao;
Expand Down Expand Up @@ -644,7 +652,17 @@ public KubernetesClusterResponse createKubernetesClusterResponse(long kubernetes
return response;
}

private void validateEndpointUrl() {
String csUrl = ApiServiceConfiguration.ApiServletPath.value();
if (csUrl == null || csUrl.contains("localhost")) {
String error = String.format("Global setting %s has to be set to the Management Server's API end point",
ApiServiceConfiguration.ApiServletPath.key());
throw new InvalidParameterValueException(error);
}
}

private void validateKubernetesClusterCreateParameters(final CreateKubernetesClusterCmd cmd) throws CloudRuntimeException {
validateEndpointUrl();
final String name = cmd.getName();
final Long zoneId = cmd.getZoneId();
final Long kubernetesVersionId = cmd.getKubernetesVersionId();
Expand Down Expand Up @@ -927,6 +945,8 @@ private void validateKubernetesClusterScaleParameters(ScaleKubernetesClusterCmd

private void validateKubernetesClusterUpgradeParameters(UpgradeKubernetesClusterCmd cmd) {
// Validate parameters
validateEndpointUrl();

final Long kubernetesClusterId = cmd.getId();
final Long upgradeVersionId = cmd.getKubernetesVersionId();
if (kubernetesClusterId == null || kubernetesClusterId < 1L) {
Expand Down Expand Up @@ -1093,13 +1113,39 @@ public boolean startKubernetesCluster(long kubernetesClusterId, boolean onCreate
startWorker = ComponentContext.inject(startWorker);
if (onCreate) {
// Start for Kubernetes cluster in 'Created' state
Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
String[] keys = getServiceUserKeys(owner);
startWorker.setKeys(keys);
return startWorker.startKubernetesClusterOnCreate();
} else {
// Start for Kubernetes cluster in 'Stopped' state. Resources are already provisioned, just need to be started
return startWorker.startStoppedKubernetesCluster();
}
}

private String[] getServiceUserKeys(Account owner) {
if (owner == null) {
owner = CallContext.current().getCallingAccount();
}
String username = owner.getAccountName() + "-" + KUBEADMIN_ACCOUNT_NAME;

@yadvr yadvr Aug 3, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we assume/create a service account/user globally, or is it per account now (a service user just for k8s use-case)?

@yadvr yadvr Aug 3, 2021

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nevermind read the code, looks like we create if it doesn't exist.

@davidjumani davidjumani Aug 3, 2021

Copy link
Copy Markdown
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Per account basis just for kube

UserAccount kubeadmin = accountService.getActiveUserAccount(username, owner.getDomainId());
String[] keys = null;
if (kubeadmin == null) {
User kube = userDao.persist(new UserVO(owner.getAccountId(), username, UUID.randomUUID().toString(), owner.getAccountName(),
KUBEADMIN_ACCOUNT_NAME, "kubeadmin", null, UUID.randomUUID().toString(), User.Source.UNKNOWN));
keys = accountService.createApiKeyAndSecretKey(kube.getId());
} else {
String apiKey = kubeadmin.getApiKey();
String secretKey = kubeadmin.getSecretKey();
if (Strings.isNullOrEmpty(apiKey) || Strings.isNullOrEmpty(secretKey)) {
keys = accountService.createApiKeyAndSecretKey(kubeadmin.getId());
} else {
keys = new String[]{apiKey, secretKey};
}
}
return keys;
}

@Override
public boolean stopKubernetesCluster(long kubernetesClusterId) throws CloudRuntimeException {
if (!KubernetesServiceEnabled.value()) {
Expand Down Expand Up @@ -1240,9 +1286,12 @@ public boolean upgradeKubernetesCluster(UpgradeKubernetesClusterCmd cmd) throws
logAndThrow(Level.ERROR, "Kubernetes Service plugin is disabled");
}
validateKubernetesClusterUpgradeParameters(cmd);
KubernetesClusterVO kubernetesCluster = kubernetesClusterDao.findById(cmd.getId());
Account owner = accountService.getActiveAccountById(kubernetesCluster.getAccountId());
String[] keys = getServiceUserKeys(owner);
KubernetesClusterUpgradeWorker upgradeWorker =
new KubernetesClusterUpgradeWorker(kubernetesClusterDao.findById(cmd.getId()),
kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this);
kubernetesSupportedVersionDao.findById(cmd.getKubernetesVersionId()), this, keys);
upgradeWorker = ComponentContext.inject(upgradeWorker);
return upgradeWorker.upgradeCluster();
}
Expand Down
1 change: 1 addition & 0 deletions ...bernetes-service/src/main/java/com/cloud/kubernetes/cluster/KubernetesClusterService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ public interface KubernetesClusterService extends PluggableService, Configurable
static final String MIN_KUBERNETES_VERSION_HA_SUPPORT = "1.16.0";
static final int MIN_KUBERNETES_CLUSTER_NODE_CPU = 2;
static final int MIN_KUBERNETES_CLUSTER_NODE_RAM_SIZE = 2048;
static final String KUBEADMIN_ACCOUNT_NAME = "kubeadmin";

static final ConfigKey<Boolean> KubernetesServiceEnabled = new ConfigKey<Boolean>("Advanced", Boolean.class,
"cloud.kubernetes.service.enabled",
Expand Down
119 changes: 119 additions & 0 deletions ...c/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterActionWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,9 @@

package com.cloud.kubernetes.cluster.actionworkers;

import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
Expand All @@ -28,6 +30,7 @@

import org.apache.cloudstack.api.ApiConstants;
import org.apache.cloudstack.ca.CAManager;
import org.apache.cloudstack.config.ApiServiceConfiguration;
import org.apache.cloudstack.engine.orchestration.service.NetworkOrchestrationService;
import org.apache.cloudstack.framework.config.dao.ConfigurationDao;
import org.apache.commons.collections.CollectionUtils;
Expand All @@ -50,6 +53,7 @@
import com.cloud.network.IpAddress;
import com.cloud.network.IpAddressManager;
import com.cloud.network.Network;
import com.cloud.network.Network.GuestType;
import com.cloud.network.NetworkModel;
import com.cloud.network.dao.NetworkDao;
import com.cloud.service.dao.ServiceOfferingDao;
Expand All @@ -70,6 +74,7 @@
import com.cloud.utils.exception.CloudRuntimeException;
import com.cloud.utils.fsm.NoTransitionException;
import com.cloud.utils.fsm.StateMachine2;
import com.cloud.utils.ssh.SshHelper;
import com.cloud.vm.UserVmService;
import com.cloud.vm.dao.UserVmDao;
import com.google.common.base.Strings;
Expand Down Expand Up @@ -127,12 +132,22 @@ public class KubernetesClusterActionWorker {
protected String publicIpAddress;
protected int sshPort;


protected final String deploySecretsScriptFilename = "deploy-cloudstack-secret";
protected final String deployProviderScriptFilename = "deploy-provider";
protected final String scriptPath = "/opt/bin/";
protected File deploySecretsScriptFile;
protected File deployProviderScriptFile;
protected KubernetesClusterManagerImpl manager;
protected String[] keys;

protected KubernetesClusterActionWorker(final KubernetesCluster kubernetesCluster, final KubernetesClusterManagerImpl clusterManager) {
this.kubernetesCluster = kubernetesCluster;
this.kubernetesClusterDao = clusterManager.kubernetesClusterDao;
this.kubernetesClusterDetailsDao = clusterManager.kubernetesClusterDetailsDao;
this.kubernetesClusterVmMapDao = clusterManager.kubernetesClusterVmMapDao;
this.kubernetesSupportedVersionDao = clusterManager.kubernetesSupportedVersionDao;
this.manager = clusterManager;
}

protected void init() {
Expand Down Expand Up @@ -380,4 +395,108 @@ protected boolean stateTransitTo(long kubernetesClusterId, KubernetesCluster.Eve
return false;
}
}

protected boolean createCloudStackSecret(String[] keys) {
File pkFile = getManagementServerSshPublicKeyFile();
Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
publicIpAddress = publicIpSshPort.first();
sshPort = publicIpSshPort.second();

try {
final String command = String.format("sudo %s/%s -u '%s' -k '%s' -s '%s'",
scriptPath, deploySecretsScriptFilename, ApiServiceConfiguration.ApiServletPath.value(), keys[0], keys[1]);
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
pkFile, null, command, 10000, 10000, 60000);
return result.first();
} catch (Exception e) {
String msg = String.format("Failed to add cloudstack-secret to Kubernetes cluster: %s", kubernetesCluster.getName());
LOGGER.warn(msg, e);
}
return false;
}

protected File retrieveScriptFile(String filename) {
File file = null;
try {
String data = readResourceFile("/script/" + filename);
file = File.createTempFile(filename, ".sh");
BufferedWriter writer = new BufferedWriter(new FileWriter(file));
writer.write(data);
writer.close();
} catch (IOException e) {
logAndThrow(Level.ERROR, String.format("Kubernetes Cluster %s : Failed to to fetch script %s",
kubernetesCluster.getName(), filename), e);
}
return file;
}

protected void retrieveScriptFiles() {
deploySecretsScriptFile = retrieveScriptFile(deploySecretsScriptFilename);
deployProviderScriptFile = retrieveScriptFile(deployProviderScriptFilename);
}

protected void copyScripts(String nodeAddress, final int sshPort) {
try {
SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
"~/", deploySecretsScriptFile.getAbsolutePath(), "0755");
SshHelper.scpTo(nodeAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
"~/", deployProviderScriptFile.getAbsolutePath(), "0755");
String cmdStr = String.format("sudo mv ~/%s %s/%s", deploySecretsScriptFile.getName(), scriptPath, deploySecretsScriptFilename);
SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
cmdStr, 10000, 10000, 10 * 60 * 1000);
cmdStr = String.format("sudo mv ~/%s %s/%s", deployProviderScriptFile.getName(), scriptPath, deployProviderScriptFilename);
SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER, sshKeyFile, null,
cmdStr, 10000, 10000, 10 * 60 * 1000);
} catch (Exception e) {
throw new CloudRuntimeException(e);
}
}

protected boolean deployProvider() {
Network network = networkDao.findById(kubernetesCluster.getNetworkId());
// Since the provider creates IP addresses, don't deploy it unless the underlying network supports it
if (network.getGuestType() != GuestType.Isolated) {
logMessage(Level.INFO, String.format("Skipping adding the provider as %s is not on an isolated network",
kubernetesCluster.getName()), null);
return true;
Comment thread
davidjumani marked this conversation as resolved.
}
File pkFile = getManagementServerSshPublicKeyFile();
Pair<String, Integer> publicIpSshPort = getKubernetesClusterServerIpSshPort(null);
publicIpAddress = publicIpSshPort.first();
sshPort = publicIpSshPort.second();

try {
String command = String.format("sudo %s/%s", scriptPath, deployProviderScriptFilename);
Pair<Boolean, String> result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
pkFile, null, command, 10000, 10000, 60000);

// Maybe the file isn't present. Try and copy it
if (!result.first()) {
logMessage(Level.INFO, "Provider files missing. Adding them now", null);
retrieveScriptFiles();
copyScripts(publicIpAddress, sshPort);

if (!createCloudStackSecret(keys)) {
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup keys for Kubernetes cluster %s",
kubernetesCluster.getName()), kubernetesCluster.getId(), KubernetesCluster.Event.OperationFailed);
}

// If at first you don't succeed ...
result = SshHelper.sshExecute(publicIpAddress, sshPort, CLUSTER_NODE_VM_USER,
pkFile, null, command, 10000, 10000, 60000);
if (!result.first()) {
throw new CloudRuntimeException(result.second());
}
}
return true;
} catch (Exception e) {
String msg = String.format("Failed to deploy kubernetes provider: %s : %s", kubernetesCluster.getName(), e.getMessage());
logAndThrow(Level.ERROR, msg);
return false;
}
}

public void setKeys(String[] keys) {
this.keys = keys;
}
}
1 change: 1 addition & 0 deletions ...rc/main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterStartWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -571,6 +571,7 @@ public boolean startKubernetesClusterOnCreate() {
if (!isKubernetesClusterDashboardServiceRunning(true, startTimeoutTime)) {
logTransitStateAndThrow(Level.ERROR, String.format("Failed to setup Kubernetes cluster : %s in usable state as unable to get Dashboard service running for the cluster", kubernetesCluster.getName()), kubernetesCluster.getId(),KubernetesCluster.Event.OperationFailed);
}
deployProvider();
stateTransitTo(kubernetesCluster.getId(), KubernetesCluster.Event.OperationSucceeded);
return true;
}
Expand Down
6 changes: 5 additions & 1 deletion .../main/java/com/cloud/kubernetes/cluster/actionworkers/KubernetesClusterUpgradeWorker.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,9 +50,11 @@ public class KubernetesClusterUpgradeWorker extends KubernetesClusterActionWorke

public KubernetesClusterUpgradeWorker(final KubernetesCluster kubernetesCluster,
final KubernetesSupportedVersion upgradeVersion,
final KubernetesClusterManagerImpl clusterManager) {
final KubernetesClusterManagerImpl clusterManager,
final String[] keys) {
super(kubernetesCluster, clusterManager);
this.upgradeVersion = upgradeVersion;
this.keys = keys;
}

private void retrieveUpgradeScriptFile() {
Expand Down Expand Up @@ -110,6 +112,8 @@ private void upgradeKubernetesClusterNodes() {
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, upgrade action timed out", kubernetesCluster.getName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, null);
}
try {
int port = (sshPort == CLUSTER_NODES_DEFAULT_START_SSH_PORT) ? sshPort + i : sshPort;
deployProvider();
result = runInstallScriptOnVM(vm, i);
} catch (Exception e) {
logTransitStateDetachIsoAndThrow(Level.ERROR, String.format("Failed to upgrade Kubernetes cluster : %s, unable to upgrade Kubernetes node on VM : %s", kubernetesCluster.getName(), vm.getDisplayName()), kubernetesCluster, clusterVMs, KubernetesCluster.Event.OperationFailed, e);
Expand Down
4 changes: 4 additions & 0 deletions plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node-add.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ write-files:
done <<< "$output"
setup_complete=true
fi
if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
mkdir -p /opt/provider
cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
fi
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
eject "${iso_drive_path}"
Expand Down
4 changes: 4 additions & 0 deletions plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-control-node.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ write-files:
fi
mkdir -p "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
cp ${BINARIES_DIR}/*.yaml "${K8S_CONFIG_SCRIPTS_COPY_DIR}"
if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
mkdir -p /opt/provider
cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
fi
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
eject "${iso_drive_path}"
Expand Down
4 changes: 4 additions & 0 deletions plugins/integrations/kubernetes-service/src/main/resources/conf/k8s-node.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,10 @@ write-files:
done <<< "$output"
setup_complete=true
fi
if [ -e "${BINARIES_DIR}/provider.yaml" ]; then
mkdir -p /opt/provider
cp "${BINARIES_DIR}/provider.yaml" /opt/provider/provider.yaml
fi
umount "${ISO_MOUNT_DIR}" && rmdir "${ISO_MOUNT_DIR}"
if [ "$EJECT_ISO_FROM_OS" = true ] && [ "$iso_drive_path" != "" ]; then
eject "${iso_drive_path}"
Expand Down
Loading