Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix proxy 2 #1542

Open
wants to merge 5 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
import io.streamnative.pulsar.handlers.mqtt.common.utils.ConfigurationUtils;
import io.streamnative.pulsar.handlers.mqtt.proxy.MQTTProxyConfiguration;
import io.streamnative.pulsar.handlers.mqtt.proxy.MQTTProxyService;
import io.streamnative.pulsar.handlers.mqtt.proxy.MQTTProxyServiceConfig;
import java.net.InetSocketAddress;
import java.util.Map;
import java.util.concurrent.Executors;
Expand All @@ -57,6 +58,9 @@ public class MQTTProtocolHandler implements ProtocolHandler {
@Getter
private String bindAddress;

@Getter
private String advertisedAddress;

private MQTTProxyService proxyService;

@Getter
Expand All @@ -77,10 +81,12 @@ public boolean accept(String protocol) {
@Override
public void initialize(ServiceConfiguration conf) throws Exception {
// init config
mqttConfig = ConfigurationUtils.create(conf.getProperties(), MQTTServerConfiguration.class);
this.mqttConfig = ConfigurationUtils.create(conf.getProperties(), MQTTServerConfiguration.class);
// We have to enable ack batch message individual.
mqttConfig.setAcknowledgmentAtBatchIndexLevelEnabled(true);
this.mqttConfig.setAcknowledgmentAtBatchIndexLevelEnabled(true);
this.bindAddress = ServiceConfigurationUtils.getDefaultOrConfiguredAddress(mqttConfig.getBindAddress());
this.advertisedAddress =
ServiceConfigurationUtils.getDefaultOrConfiguredAddress(mqttConfig.getAdvertisedAddress());
}

@Override
Expand All @@ -97,11 +103,10 @@ public void start(BrokerService brokerService) {
mqttService = new MQTTService(brokerService, mqttConfig);
if (mqttConfig.isMqttProxyEnabled() || mqttConfig.isMqttProxyEnable()) {
try {
MQTTProxyConfiguration proxyConfig =
ConfigurationUtils.create(mqttConfig.getProperties(), MQTTProxyConfiguration.class);
proxyService = new MQTTProxyService(brokerService, proxyConfig);
final MQTTProxyServiceConfig config = initProxyConfig();
proxyService = new MQTTProxyService(config);
proxyService.start();
log.info("Start MQTT proxy service at port: {}", proxyConfig.getMqttProxyPort());
log.info("Start MQTT proxy service at port: {}", config.getProxyConfiguration().getMqttProxyPort());
} catch (Exception ex) {
log.error("Failed to start MQTT proxy service.", ex);
}
Expand All @@ -114,6 +119,20 @@ public void start(BrokerService brokerService) {
MopVersion.getBuildTime());
}

private MQTTProxyServiceConfig initProxyConfig() throws Exception {
MQTTProxyConfiguration proxyConfig =
ConfigurationUtils.create(mqttConfig.getProperties(), MQTTProxyConfiguration.class);
MQTTProxyServiceConfig config = new MQTTProxyServiceConfig();
config.setBindAddress(bindAddress);
config.setAdvertisedAddress(advertisedAddress);
config.setProxyConfiguration(proxyConfig);
config.setLocalMetadataStore(brokerService.pulsar().getLocalMetadataStore());
config.setConfigMetadataStore(brokerService.pulsar().getConfigurationMetadataStore());
config.setPulsarClient(brokerService.pulsar().getClient());
config.setPulsarResources(brokerService.pulsar().getPulsarResources());
return config;
}

@Override
public Map<InetSocketAddress, ChannelInitializer<SocketChannel>> newChannelInitializers() {
checkArgument(mqttConfig != null);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -102,14 +102,14 @@ public MQTTService(BrokerService brokerService, MQTTServerConfiguration serverCo
this.metricsProvider = new MQTTMetricsProvider(metricsCollector);
this.pulsarService.addPrometheusRawMetricsProvider(metricsProvider);
this.authenticationService = serverConfiguration.isMqttAuthenticationEnabled()
? new MQTTAuthenticationService(brokerService,
? new MQTTAuthenticationService(brokerService.getAuthenticationService(),
serverConfiguration.getMqttAuthenticationMethods()) : null;
this.connectionManager = new MQTTConnectionManager(pulsarService.getAdvertisedAddress());
this.subscriptionManager = new MQTTSubscriptionManager();
if (getServerConfiguration().isMqttProxyEnabled()) {
this.eventCenter = new DisableEventCenter();
} else {
this.eventCenter = new PulsarEventCenterImpl(brokerService,
this.eventCenter = new PulsarEventCenterImpl(brokerService.pulsar().getConfigurationMetadataStore(),
serverConfiguration.getEventCenterCallbackPoolThreadNum());
}
this.eventService = new DisabledSystemEventService();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
import org.apache.pulsar.broker.authentication.AuthenticationProvider;
import org.apache.pulsar.broker.authentication.AuthenticationService;
import org.apache.pulsar.broker.service.BrokerService;

/**
* MQTT authentication service.
Expand All @@ -45,8 +44,8 @@ public class MQTTAuthenticationService {
@Getter
private final Map<String, AuthenticationProvider> authenticationProviders;

public MQTTAuthenticationService(BrokerService brokerService, List<String> authenticationMethods) {
this.authenticationService = brokerService.getAuthenticationService();
public MQTTAuthenticationService(AuthenticationService authenticationService, List<String> authenticationMethods) {
this.authenticationService = authenticationService;
this.authenticationProviders = getAuthenticationProviders(authenticationMethods);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.common.util.OrderedExecutor;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.pulsar.broker.service.BrokerService;
import org.apache.pulsar.metadata.api.MetadataStore;
import org.apache.pulsar.metadata.api.Notification;

@Slf4j
Expand All @@ -29,13 +29,12 @@ public class PulsarEventCenterImpl implements Consumer<Notification>, PulsarEven
private final OrderedExecutor callbackExecutor;

@SuppressWarnings("UnstableApiUsage")
public PulsarEventCenterImpl(BrokerService brokerService, int poolThreadNum) {
public PulsarEventCenterImpl(MetadataStore configurationMetadataStore, int poolThreadNum) {
this.listeners = new CopyOnWriteArrayList<>();
this.callbackExecutor = OrderedExecutor.newBuilder()
.numThreads(poolThreadNum)
.name("mqtt-notification-workers").build();
brokerService.getPulsar()
.getConfigurationMetadataStore().registerListener(this);
configurationMetadataStore.registerListener(this);
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,19 +16,22 @@
import com.github.benmanes.caffeine.cache.AsyncLoadingCache;
import com.github.benmanes.caffeine.cache.Caffeine;
import com.google.common.annotations.Beta;
import io.netty.util.concurrent.DefaultThreadFactory;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import lombok.extern.slf4j.Slf4j;
import org.apache.bookkeeper.common.util.JsonUtil;
import org.apache.pulsar.broker.PulsarServerException;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.resources.PulsarResources;
import org.apache.pulsar.broker.systopic.SystemTopicClient;
import org.apache.pulsar.client.api.Message;
import org.apache.pulsar.client.api.MessageId;
import org.apache.pulsar.client.api.PulsarClient;
import org.apache.pulsar.client.api.PulsarClientException;
import org.apache.pulsar.client.util.RetryUtil;
import org.apache.pulsar.common.naming.NamespaceName;
Expand All @@ -48,29 +51,28 @@ public class SystemTopicBasedSystemEventService implements SystemEventService {
private static final long CACHE_EXPIRE_TIME_MILLIS = TimeUnit.MINUTES.toMillis(10);

private static final String WRITER_KEY = "writer";
private final PulsarService pulsarService;
private final PulsarResources pulsarResources;
private final SystemTopicClient<MqttEvent> systemTopicClient;
private final List<EventListener> listeners;
private volatile SystemTopicClient.Reader<MqttEvent> reader;
private final AtomicBoolean initReader = new AtomicBoolean(false);
private final AtomicInteger maxRetry = new AtomicInteger(0);

private final ScheduledExecutorService executor;
private final AsyncLoadingCache<String, SystemTopicClient.Writer<MqttEvent>> writerCaches;

public SystemTopicBasedSystemEventService(PulsarService pulsarService) {
this.pulsarService = pulsarService;
try {
this.systemTopicClient = new MQTTEventSystemTopicClient(pulsarService.getClient(), SYSTEM_EVENT_TOPIC);
} catch (PulsarServerException e) {
throw new IllegalStateException(e);
}
public SystemTopicBasedSystemEventService(PulsarResources pulsarResources, PulsarClient client) {
this.pulsarResources = pulsarResources;
this.systemTopicClient = new MQTTEventSystemTopicClient(client, SYSTEM_EVENT_TOPIC);
this.listeners = new ArrayList<>();
writerCaches = Caffeine.newBuilder()
.expireAfterAccess(CACHE_EXPIRE_TIME_MILLIS, TimeUnit.MILLISECONDS)
.removalListener((k, v, c) -> {
((SystemTopicClient.Writer<MqttEvent>) v).closeAsync();
})
.buildAsync((key, executor) -> systemTopicClient.newWriterAsync());
executor = Executors.newSingleThreadScheduledExecutor(
new DefaultThreadFactory("mop_system_topic_client"));
}

@Override
Expand Down Expand Up @@ -134,22 +136,21 @@ protected CompletableFuture<SystemTopicClient.Reader<MqttEvent>> createReader()
Backoff backoff = new Backoff(1, TimeUnit.SECONDS,
3, TimeUnit.SECONDS,
10, TimeUnit.SECONDS);
RetryUtil.retryAsynchronously(systemTopicClient::newReaderAsync, backoff, pulsarService.getExecutor(), result);
RetryUtil.retryAsynchronously(systemTopicClient::newReaderAsync, backoff, executor, result);
return result;
}

@Override
public void start() {
CompletableFuture<Boolean> checkNamespaceFuture = pulsarService
.getPulsarResources()
CompletableFuture<Boolean> checkNamespaceFuture = pulsarResources
.getNamespaceResources()
.namespaceExistsAsync(NamespaceName.SYSTEM_NAMESPACE);
checkNamespaceFuture.thenAccept(ret -> {
if (ret) {
startReader();
} else {
if (maxRetry.incrementAndGet() < 10) {
pulsarService.getExecutor().schedule(this::start, 1, TimeUnit.SECONDS);
executor.schedule(this::start, 1, TimeUnit.SECONDS);
}
}
}).exceptionally(ex -> {
Expand All @@ -161,6 +162,7 @@ public void start() {
@Override
public void close() {
closeReader();
executor.shutdownNow();
}

private void startReader() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import org.apache.commons.lang3.tuple.Pair;
import org.apache.pulsar.broker.PulsarService;
import org.apache.pulsar.broker.namespace.LookupOptions;
import org.apache.pulsar.broker.namespace.NamespaceService;
import org.apache.pulsar.broker.service.BrokerServiceException;
import org.apache.pulsar.broker.service.Subscription;
import org.apache.pulsar.broker.service.Topic;
Expand Down Expand Up @@ -197,14 +198,15 @@ public static TopicFilter getTopicFilter(String mqttTopicFilter) {
}

public static CompletableFuture<List<String>> asyncGetTopicListFromTopicSubscription(String topicFilter,
String defaultTenant, String defaultNamespace, PulsarService pulsarService, String defaultTopicDomain) {
String defaultTenant, String defaultNamespace,
NamespaceService namespaceService, String defaultTopicDomain) {
if (MqttUtils.isRegexFilter(topicFilter)) {
TopicFilter filter = PulsarTopicUtils.getTopicFilter(topicFilter);
Pair<TopicDomain, NamespaceName> domainNamespacePair =
PulsarTopicUtils
.getTopicDomainAndNamespaceFromTopicFilter(topicFilter, defaultTenant,
defaultNamespace, defaultTopicDomain);
return pulsarService.getNamespaceService().getListOfTopics(
return namespaceService.getListOfTopics(
domainNamespacePair.getRight(), domainNamespacePair.getLeft() == TopicDomain.persistent
? CommandGetTopicsOfNamespace.Mode.PERSISTENT
: CommandGetTopicsOfNamespace.Mode.NON_PERSISTENT).thenCompose(topics ->
Expand Down
Loading
Loading