Basically, with the following code, the form will not submit unless the text_format's format field has a selection made by the client/user. (When tied into the #state of the checkbox).
$form['mail_settings_fieldset'] = [
'#type' => 'fieldset',
'#title' => $this->t('Attendee confirmation email settings'),
];
$form['mail_settings_fieldset']['signupSendMail'] = [
'#type' => 'checkbox',
'#title' => t('Select if you wish to send an email to the client, when a new signup is submitted'),
'#default_value' => (isset($signupSendMail)) ? $signupSendMail : 0,
];
$form['mail_settings_fieldset']['subject'] = [
'#type' => 'textfield',
'#title' => $this->t('Subject'),
'#maxlength' => 254,
'#default_value' => (isset($subject)) ? $subject : '',
'#states' => [
'required' => [
':input[name="signupSendMail"]' => [
'checked' => true,
],
],
],
];
$form['mail_settings_fieldset']['message'] = [
'#type' => 'text_format',
'#title' => $this->t('Message'),
'#format' => (isset($format)) ? $format : 'basic_html',
'#allowed_formats' => [
'basic_html',
'restricted_html',
'full_html',
],
'#default_value' => (isset($body)) ? $body : '',
'#description' => t('Please make sure that all urls are absolute.'),
'#states' => [
'required' => [
':input[name="signupSendMail"]' => [
'checked' => true,
],
],
],
];
$form['actions'] = [
'#type' => 'actions',
];
$form['actions']['submit'] = [
'#type' => 'submit',
'#value' => $this->t('Configure signup'),
];